From 43d928ead853089335a92f039522b5cc5b42141b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 22 Dec 2025 19:13:06 -0700 Subject: [PATCH 1/8] skip test --- .github/workflows/pr_build_macos.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 4c955346c7..da06bff87b 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -131,6 +131,15 @@ jobs: - name: "sql" value: | org.apache.spark.sql.CometToPrettyStringSuite + + exclude: + # Skip fuzz suite for Spark 4.0 + # https://github.com/apache/datafusion-comet/issues/2965 + - profile: + name: "Spark 4.0, JDK 17, Scala 2.13" + suite: + name: "fuzz" + fail-fast: false name: ${{ matrix.os }}/${{ matrix.profile.name }} [${{ matrix.suite.name }}] runs-on: ${{ matrix.os }} From f214ba3e8adbcb50f07b7c538c9f132559ddaea7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Dec 2025 08:44:11 -0700 Subject: [PATCH 2/8] refactor --- .../apache/comet/rules/CometScanRule.scala | 114 ++++++++++++------ 1 file changed, 79 insertions(+), 35 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 01e385b0ae..9b4948e26a 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -42,7 +42,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, CometNativeException, DataTypeSupport} import org.apache.comet.CometConf._ -import org.apache.comet.CometSparkSessionExtensions.{hasExplainInfo, isCometLoaded, withInfo, withInfos} +import org.apache.comet.CometSparkSessionExtensions.{isCometLoaded, withInfo, withInfos} import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.iceberg.{CometIcebergNativeScanMetadata, IcebergReflection} import org.apache.comet.objectstore.NativeConfig @@ -145,21 +145,9 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com if (!CometScanExec.isFileFormatSupported(r.fileFormat)) { return withInfo(scanExec, s"Unsupported file format ${r.fileFormat}") } + val hadoopConf = r.sparkSession.sessionState.newHadoopConfWithOptions(r.options) - var scanImpl = COMET_NATIVE_SCAN_IMPL.get() - - val hadoopConf = scanExec.relation.sparkSession.sessionState - .newHadoopConfWithOptions(scanExec.relation.options) - - // if scan is auto then pick the best available scan - if (scanImpl == SCAN_AUTO) { - scanImpl = selectScan(scanExec, r.partitionSchema, hadoopConf) - } - - if (scanImpl == SCAN_NATIVE_DATAFUSION && !CometNativeScan.isSupported(scanExec)) { - return scanExec - } - + // TODO is this restriction valid for all native scan types? val possibleDefaultValues = getExistenceDefaultValues(scanExec.requiredSchema) if (possibleDefaultValues.exists(d => { d != null && (d.isInstanceOf[ArrayBasedMapData] || d @@ -173,22 +161,18 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com "Full native scan disabled because nested types for default values are not supported") } - if (encryptionEnabled(hadoopConf) && scanImpl != CometConf.SCAN_NATIVE_COMET) { - if (!isEncryptionConfigSupported(hadoopConf)) { - withInfo(scanExec, s"$scanImpl does not support encryption") - } - } - - // check that schema is supported - checkSchema(scanExec, scanImpl, r) - - if (hasExplainInfo(scanExec)) { - // could not accelerate, and plan is already tagged with fallback reasons - scanExec - } else { - // this is confusing, but we always insert a CometScanExec here, which may replaced - // with a CometNativeExec when CometExecRule runs, depending on the scanImpl value. - CometScanExec(scanExec, session, scanImpl) + COMET_NATIVE_SCAN_IMPL.get() match { + case SCAN_AUTO => + nativeDataFusionScan(session, scanExec, r, hadoopConf) + .orElse(nativeIcebergCompatScan(session, scanExec, r, hadoopConf)) + .orElse(nativeCometScan(session, scanExec, r, hadoopConf)) + .getOrElse(scanExec) + case SCAN_NATIVE_DATAFUSION => + nativeDataFusionScan(session, scanExec, r, hadoopConf).getOrElse(scanExec) + case SCAN_NATIVE_ICEBERG_COMPAT => + nativeIcebergCompatScan(session, scanExec, r, hadoopConf).getOrElse(scanExec) + case SCAN_NATIVE_COMET => + nativeCometScan(session, scanExec, r, hadoopConf).getOrElse(scanExec) } case _ => @@ -196,6 +180,56 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com } } + private def nativeDataFusionScan( + session: SparkSession, + scanExec: FileSourceScanExec, + r: HadoopFsRelation, + hadoopConf: Configuration): Option[SparkPlan] = { + if (!CometNativeScan.isSupported(scanExec)) { + return None + } + if (encryptionEnabled(hadoopConf)) { + if (!isEncryptionConfigSupported(hadoopConf)) { + withInfo(scanExec, s"$SCAN_NATIVE_DATAFUSION does not support encryption") + return None + } + } + if (!isSchemaSupported(scanExec, SCAN_NATIVE_DATAFUSION, r)) { + return None + } + // TODO enable SCAN_NATIVE_DATAFUSION for auto scan + // Some(CometScanExec(scanExec, session, SCAN_NATIVE_DATAFUSION)) + None + } + + private def nativeIcebergCompatScan( + session: SparkSession, + scanExec: FileSourceScanExec, + r: HadoopFsRelation, + hadoopConf: Configuration): Option[SparkPlan] = { + if (encryptionEnabled(hadoopConf)) { + if (!isEncryptionConfigSupported(hadoopConf)) { + withInfo(scanExec, s"$SCAN_NATIVE_ICEBERG_COMPAT does not support encryption") + return None + } + } + if (!isSchemaSupported(scanExec, SCAN_NATIVE_ICEBERG_COMPAT, r)) { + return None + } + Some(CometScanExec(scanExec, session, SCAN_NATIVE_ICEBERG_COMPAT)) + } + + private def nativeCometScan( + session: SparkSession, + scanExec: FileSourceScanExec, + r: HadoopFsRelation, + hadoopConf: Configuration): Option[SparkPlan] = { + if (!isSchemaSupported(scanExec, SCAN_NATIVE_COMET, r)) { + return None + } + Some(CometScanExec(scanExec, session, SCAN_NATIVE_COMET)) + } + private def transformV2Scan(scanExec: BatchScanExec): SparkPlan = { scanExec.scan match { @@ -612,20 +646,30 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com private def isDynamicPruningFilter(e: Expression): Boolean = e.exists(_.isInstanceOf[PlanExpression[_]]) - def checkSchema(scanExec: FileSourceScanExec, scanImpl: String, r: HadoopFsRelation): Unit = { + private def isSchemaSupported( + scanExec: FileSourceScanExec, + scanImpl: String, + r: HadoopFsRelation): Boolean = { val fallbackReasons = new ListBuffer[String]() val typeChecker = CometScanTypeChecker(scanImpl) val schemaSupported = typeChecker.isSchemaSupported(scanExec.requiredSchema, fallbackReasons) if (!schemaSupported) { - withInfo(scanExec, s"Unsupported schema ${scanExec.requiredSchema} for $scanImpl") + withInfo( + scanExec, + s"Unsupported schema ${scanExec.requiredSchema} for $scanImpl: ${fallbackReasons.mkString(", ")}") + return false } val partitionSchemaSupported = typeChecker.isSchemaSupported(r.partitionSchema, fallbackReasons) if (!partitionSchemaSupported) { - fallbackReasons += s"Unsupported partitioning schema ${r.partitionSchema} for $scanImpl" + withInfo( + scanExec, + s"Unsupported partitioning schema ${scanExec.requiredSchema} for $scanImpl: ${fallbackReasons + .mkString(", ")}") + return false } - withInfos(scanExec, fallbackReasons.toSet) + true } } From 11b2e67eca027c331867800bd92881e17bc24123 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Dec 2025 08:59:08 -0700 Subject: [PATCH 3/8] refactor --- .../apache/comet/rules/CometScanRule.scala | 20 +++++++++---------- .../org/apache/comet/CometFuzzTestBase.scala | 3 ++- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 9b4948e26a..cb75981edf 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -188,18 +188,14 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com if (!CometNativeScan.isSupported(scanExec)) { return None } - if (encryptionEnabled(hadoopConf)) { - if (!isEncryptionConfigSupported(hadoopConf)) { - withInfo(scanExec, s"$SCAN_NATIVE_DATAFUSION does not support encryption") - return None - } + if (encryptionEnabled(hadoopConf) && !isEncryptionConfigSupported(hadoopConf)) { + withInfo(scanExec, s"$SCAN_NATIVE_DATAFUSION does not support encryption") + return None } if (!isSchemaSupported(scanExec, SCAN_NATIVE_DATAFUSION, r)) { return None } - // TODO enable SCAN_NATIVE_DATAFUSION for auto scan - // Some(CometScanExec(scanExec, session, SCAN_NATIVE_DATAFUSION)) - None + Some(CometScanExec(scanExec, session, SCAN_NATIVE_DATAFUSION)) } private def nativeIcebergCompatScan( @@ -657,7 +653,8 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com if (!schemaSupported) { withInfo( scanExec, - s"Unsupported schema ${scanExec.requiredSchema} for $scanImpl: ${fallbackReasons.mkString(", ")}") + s"Unsupported schema ${scanExec.requiredSchema} " + + s"for $scanImpl: ${fallbackReasons.mkString(", ")}") return false } val partitionSchemaSupported = @@ -665,8 +662,9 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com if (!partitionSchemaSupported) { withInfo( scanExec, - s"Unsupported partitioning schema ${scanExec.requiredSchema} for $scanImpl: ${fallbackReasons - .mkString(", ")}") + s"Unsupported partitioning schema ${scanExec.requiredSchema} " + + s"for $scanImpl: ${fallbackReasons + .mkString(", ")}") return false } true diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala index 74858ed614..ca4d068796 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala @@ -39,9 +39,10 @@ import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGener class CometFuzzTestBase extends CometTestBase with AdaptiveSparkPlanHelper { + /** Path for generated data files */ var filename: String = null - /** Filename for data file with deeply nested complex types */ + /** Path for generated data files with deeply nested complex types */ var complexTypesFilename: String = null /** From 16f28e5243669e5f16785c6184cfbcbbef096607 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Dec 2025 09:03:45 -0700 Subject: [PATCH 4/8] refactor --- .../main/scala/org/apache/comet/rules/CometScanRule.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index cb75981edf..9c8cc3f8de 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -203,11 +203,9 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com scanExec: FileSourceScanExec, r: HadoopFsRelation, hadoopConf: Configuration): Option[SparkPlan] = { - if (encryptionEnabled(hadoopConf)) { - if (!isEncryptionConfigSupported(hadoopConf)) { - withInfo(scanExec, s"$SCAN_NATIVE_ICEBERG_COMPAT does not support encryption") - return None - } + if (encryptionEnabled(hadoopConf) && !isEncryptionConfigSupported(hadoopConf)) { + withInfo(scanExec, s"$SCAN_NATIVE_ICEBERG_COMPAT does not support encryption") + return None } if (!isSchemaSupported(scanExec, SCAN_NATIVE_ICEBERG_COMPAT, r)) { return None From 6d1445df2fc0383303426aa77c536794af5553eb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Dec 2025 09:54:16 -0700 Subject: [PATCH 5/8] golden files --- .../approved-plans-v1_4/q1/explain.txt | 16 +- .../approved-plans-v1_4/q1/extended.txt | 10 +- .../approved-plans-v1_4/q1/simplified.txt | 8 +- .../approved-plans-v1_4/q10/explain.txt | 20 +- .../approved-plans-v1_4/q10/extended.txt | 14 +- .../approved-plans-v1_4/q10/simplified.txt | 10 +- .../approved-plans-v1_4/q11/explain.txt | 32 +- .../approved-plans-v1_4/q11/extended.txt | 20 +- .../approved-plans-v1_4/q11/simplified.txt | 16 +- .../approved-plans-v1_4/q12/explain.txt | 12 +- .../approved-plans-v1_4/q12/extended.txt | 6 +- .../approved-plans-v1_4/q12/simplified.txt | 6 +- .../approved-plans-v1_4/q13/explain.txt | 24 +- .../approved-plans-v1_4/q13/extended.txt | 12 +- .../approved-plans-v1_4/q13/simplified.txt | 12 +- .../approved-plans-v1_4/q14a/explain.txt | 32 +- .../approved-plans-v1_4/q14a/extended.txt | 116 ++-- .../approved-plans-v1_4/q14a/simplified.txt | 16 +- .../approved-plans-v1_4/q14b/explain.txt | 48 +- .../approved-plans-v1_4/q14b/extended.txt | 86 +-- .../approved-plans-v1_4/q14b/simplified.txt | 24 +- .../approved-plans-v1_4/q15/explain.txt | 16 +- .../approved-plans-v1_4/q15/extended.txt | 8 +- .../approved-plans-v1_4/q15/simplified.txt | 8 +- .../approved-plans-v1_4/q16/explain.txt | 24 +- .../approved-plans-v1_4/q16/extended.txt | 12 +- .../approved-plans-v1_4/q16/simplified.txt | 12 +- .../approved-plans-v1_4/q17/explain.txt | 24 +- .../approved-plans-v1_4/q17/extended.txt | 14 +- .../approved-plans-v1_4/q17/simplified.txt | 12 +- .../approved-plans-v1_4/q18/explain.txt | 28 +- .../approved-plans-v1_4/q18/extended.txt | 14 +- .../approved-plans-v1_4/q18/simplified.txt | 14 +- .../approved-plans-v1_4/q19/explain.txt | 26 +- .../approved-plans-v1_4/q19/extended.txt | 12 +- .../approved-plans-v1_4/q19/simplified.txt | 12 +- .../approved-plans-v1_4/q2/explain.txt | 24 +- .../approved-plans-v1_4/q2/extended.txt | 16 +- .../approved-plans-v1_4/q2/simplified.txt | 10 +- .../approved-plans-v1_4/q20/explain.txt | 12 +- .../approved-plans-v1_4/q20/extended.txt | 6 +- .../approved-plans-v1_4/q20/simplified.txt | 6 +- .../approved-plans-v1_4/q21/explain.txt | 16 +- .../approved-plans-v1_4/q21/extended.txt | 8 +- .../approved-plans-v1_4/q21/simplified.txt | 8 +- .../approved-plans-v1_4/q22/explain.txt | 16 +- .../approved-plans-v1_4/q22/extended.txt | 8 +- .../approved-plans-v1_4/q22/simplified.txt | 8 +- .../approved-plans-v1_4/q23a/explain.txt | 36 +- .../approved-plans-v1_4/q23a/extended.txt | 32 +- .../approved-plans-v1_4/q23a/simplified.txt | 18 +- .../approved-plans-v1_4/q23b/explain.txt | 40 +- .../approved-plans-v1_4/q23b/extended.txt | 44 +- .../approved-plans-v1_4/q23b/simplified.txt | 20 +- .../approved-plans-v1_4/q24a/explain.txt | 28 +- .../approved-plans-v1_4/q24a/extended.txt | 24 +- .../approved-plans-v1_4/q24a/simplified.txt | 14 +- .../approved-plans-v1_4/q24b/explain.txt | 28 +- .../approved-plans-v1_4/q24b/extended.txt | 24 +- .../approved-plans-v1_4/q24b/simplified.txt | 14 +- .../approved-plans-v1_4/q25/explain.txt | 24 +- .../approved-plans-v1_4/q25/extended.txt | 14 +- .../approved-plans-v1_4/q25/simplified.txt | 12 +- .../approved-plans-v1_4/q26/explain.txt | 20 +- .../approved-plans-v1_4/q26/extended.txt | 10 +- .../approved-plans-v1_4/q26/simplified.txt | 10 +- .../approved-plans-v1_4/q27/explain.txt | 20 +- .../approved-plans-v1_4/q27/extended.txt | 10 +- .../approved-plans-v1_4/q27/simplified.txt | 10 +- .../approved-plans-v1_4/q28/explain.txt | 24 +- .../approved-plans-v1_4/q28/extended.txt | 12 +- .../approved-plans-v1_4/q28/simplified.txt | 12 +- .../approved-plans-v1_4/q29/explain.txt | 32 +- .../approved-plans-v1_4/q29/extended.txt | 16 +- .../approved-plans-v1_4/q29/simplified.txt | 16 +- .../approved-plans-v1_4/q3/explain.txt | 14 +- .../approved-plans-v1_4/q3/extended.txt | 6 +- .../approved-plans-v1_4/q3/simplified.txt | 6 +- .../approved-plans-v1_4/q30/explain.txt | 20 +- .../approved-plans-v1_4/q30/extended.txt | 14 +- .../approved-plans-v1_4/q30/simplified.txt | 10 +- .../approved-plans-v1_4/q31/explain.txt | 28 +- .../approved-plans-v1_4/q31/extended.txt | 30 +- .../approved-plans-v1_4/q31/simplified.txt | 14 +- .../approved-plans-v1_4/q32/explain.txt | 12 +- .../approved-plans-v1_4/q32/extended.txt | 8 +- .../approved-plans-v1_4/q32/simplified.txt | 6 +- .../approved-plans-v1_4/q33/explain.txt | 28 +- .../approved-plans-v1_4/q33/extended.txt | 26 +- .../approved-plans-v1_4/q33/simplified.txt | 14 +- .../approved-plans-v1_4/q34/explain.txt | 20 +- .../approved-plans-v1_4/q34/extended.txt | 10 +- .../approved-plans-v1_4/q34/simplified.txt | 10 +- .../approved-plans-v1_4/q35/explain.txt | 20 +- .../approved-plans-v1_4/q35/extended.txt | 14 +- .../approved-plans-v1_4/q35/simplified.txt | 10 +- .../approved-plans-v1_4/q36/explain.txt | 16 +- .../approved-plans-v1_4/q36/extended.txt | 8 +- .../approved-plans-v1_4/q36/simplified.txt | 8 +- .../approved-plans-v1_4/q37/explain.txt | 16 +- .../approved-plans-v1_4/q37/extended.txt | 8 +- .../approved-plans-v1_4/q37/simplified.txt | 8 +- .../approved-plans-v1_4/q38/explain.txt | 12 +- .../approved-plans-v1_4/q38/extended.txt | 14 +- .../approved-plans-v1_4/q38/simplified.txt | 6 +- .../approved-plans-v1_4/q39a/explain.txt | 24 +- .../approved-plans-v1_4/q39a/extended.txt | 16 +- .../approved-plans-v1_4/q39a/simplified.txt | 12 +- .../approved-plans-v1_4/q39b/explain.txt | 24 +- .../approved-plans-v1_4/q39b/extended.txt | 16 +- .../approved-plans-v1_4/q39b/simplified.txt | 12 +- .../approved-plans-v1_4/q4/explain.txt | 40 +- .../approved-plans-v1_4/q4/extended.txt | 28 +- .../approved-plans-v1_4/q4/simplified.txt | 20 +- .../approved-plans-v1_4/q40/explain.txt | 20 +- .../approved-plans-v1_4/q40/extended.txt | 10 +- .../approved-plans-v1_4/q40/simplified.txt | 10 +- .../approved-plans-v1_4/q41/explain.txt | 8 +- .../approved-plans-v1_4/q41/extended.txt | 4 +- .../approved-plans-v1_4/q41/simplified.txt | 4 +- .../approved-plans-v1_4/q42/explain.txt | 14 +- .../approved-plans-v1_4/q42/extended.txt | 6 +- .../approved-plans-v1_4/q42/simplified.txt | 6 +- .../approved-plans-v1_4/q43/explain.txt | 14 +- .../approved-plans-v1_4/q43/extended.txt | 6 +- .../approved-plans-v1_4/q43/simplified.txt | 6 +- .../approved-plans-v1_4/q44/explain.txt | 12 +- .../approved-plans-v1_4/q44/extended.txt | 12 +- .../approved-plans-v1_4/q44/simplified.txt | 6 +- .../approved-plans-v1_4/q45/explain.txt | 24 +- .../approved-plans-v1_4/q45/extended.txt | 12 +- .../approved-plans-v1_4/q45/simplified.txt | 12 +- .../approved-plans-v1_4/q46/explain.txt | 24 +- .../approved-plans-v1_4/q46/extended.txt | 14 +- .../approved-plans-v1_4/q46/simplified.txt | 12 +- .../approved-plans-v1_4/q47/explain.txt | 16 +- .../approved-plans-v1_4/q47/extended.txt | 24 +- .../approved-plans-v1_4/q47/simplified.txt | 8 +- .../approved-plans-v1_4/q48/explain.txt | 20 +- .../approved-plans-v1_4/q48/extended.txt | 10 +- .../approved-plans-v1_4/q48/simplified.txt | 10 +- .../approved-plans-v1_4/q49/explain.txt | 20 +- .../approved-plans-v1_4/q49/extended.txt | 14 +- .../approved-plans-v1_4/q49/simplified.txt | 10 +- .../approved-plans-v1_4/q5/explain.txt | 24 +- .../approved-plans-v1_4/q5/extended.txt | 16 +- .../approved-plans-v1_4/q5/simplified.txt | 12 +- .../approved-plans-v1_4/q50/explain.txt | 22 +- .../approved-plans-v1_4/q50/extended.txt | 10 +- .../approved-plans-v1_4/q50/simplified.txt | 10 +- .../approved-plans-v1_4/q51/explain.txt | 8 +- .../approved-plans-v1_4/q51/extended.txt | 6 +- .../approved-plans-v1_4/q51/simplified.txt | 4 +- .../approved-plans-v1_4/q52/explain.txt | 14 +- .../approved-plans-v1_4/q52/extended.txt | 6 +- .../approved-plans-v1_4/q52/simplified.txt | 6 +- .../approved-plans-v1_4/q53/explain.txt | 16 +- .../approved-plans-v1_4/q53/extended.txt | 8 +- .../approved-plans-v1_4/q53/simplified.txt | 8 +- .../approved-plans-v1_4/q54/explain.txt | 40 +- .../approved-plans-v1_4/q54/extended.txt | 20 +- .../approved-plans-v1_4/q54/simplified.txt | 20 +- .../approved-plans-v1_4/q55/explain.txt | 14 +- .../approved-plans-v1_4/q55/extended.txt | 6 +- .../approved-plans-v1_4/q55/simplified.txt | 6 +- .../approved-plans-v1_4/q56/explain.txt | 20 +- .../approved-plans-v1_4/q56/extended.txt | 26 +- .../approved-plans-v1_4/q56/simplified.txt | 10 +- .../approved-plans-v1_4/q57/explain.txt | 16 +- .../approved-plans-v1_4/q57/extended.txt | 24 +- .../approved-plans-v1_4/q57/simplified.txt | 8 +- .../approved-plans-v1_4/q58/explain.txt | 32 +- .../approved-plans-v1_4/q58/extended.txt | 30 +- .../approved-plans-v1_4/q58/simplified.txt | 16 +- .../approved-plans-v1_4/q59/explain.txt | 26 +- .../approved-plans-v1_4/q59/extended.txt | 16 +- .../approved-plans-v1_4/q59/simplified.txt | 12 +- .../approved-plans-v1_4/q6/explain.txt | 28 +- .../approved-plans-v1_4/q6/extended.txt | 14 +- .../approved-plans-v1_4/q6/simplified.txt | 14 +- .../approved-plans-v1_4/q60/explain.txt | 20 +- .../approved-plans-v1_4/q60/extended.txt | 26 +- .../approved-plans-v1_4/q60/simplified.txt | 10 +- .../approved-plans-v1_4/q61/explain.txt | 28 +- .../approved-plans-v1_4/q61/extended.txt | 24 +- .../approved-plans-v1_4/q61/simplified.txt | 14 +- .../approved-plans-v1_4/q62/explain.txt | 20 +- .../approved-plans-v1_4/q62/extended.txt | 10 +- .../approved-plans-v1_4/q62/simplified.txt | 10 +- .../approved-plans-v1_4/q63/explain.txt | 16 +- .../approved-plans-v1_4/q63/extended.txt | 8 +- .../approved-plans-v1_4/q63/simplified.txt | 8 +- .../approved-plans-v1_4/q64/explain.txt | 68 +- .../approved-plans-v1_4/q64/extended.txt | 76 +-- .../approved-plans-v1_4/q64/simplified.txt | 34 +- .../approved-plans-v1_4/q65/explain.txt | 16 +- .../approved-plans-v1_4/q65/extended.txt | 10 +- .../approved-plans-v1_4/q65/simplified.txt | 8 +- .../approved-plans-v1_4/q66/explain.txt | 20 +- .../approved-plans-v1_4/q66/extended.txt | 18 +- .../approved-plans-v1_4/q66/simplified.txt | 10 +- .../approved-plans-v1_4/q67/explain.txt | 16 +- .../approved-plans-v1_4/q67/extended.txt | 8 +- .../approved-plans-v1_4/q67/simplified.txt | 8 +- .../approved-plans-v1_4/q68/explain.txt | 24 +- .../approved-plans-v1_4/q68/extended.txt | 14 +- .../approved-plans-v1_4/q68/simplified.txt | 12 +- .../approved-plans-v1_4/q69/explain.txt | 20 +- .../approved-plans-v1_4/q69/extended.txt | 14 +- .../approved-plans-v1_4/q69/simplified.txt | 10 +- .../approved-plans-v1_4/q7/explain.txt | 20 +- .../approved-plans-v1_4/q7/extended.txt | 10 +- .../approved-plans-v1_4/q7/simplified.txt | 10 +- .../approved-plans-v1_4/q70/explain.txt | 16 +- .../approved-plans-v1_4/q70/extended.txt | 10 +- .../approved-plans-v1_4/q70/simplified.txt | 8 +- .../approved-plans-v1_4/q71/explain.txt | 16 +- .../approved-plans-v1_4/q71/extended.txt | 12 +- .../approved-plans-v1_4/q71/simplified.txt | 8 +- .../approved-plans-v1_4/q72/explain.txt | 46 +- .../approved-plans-v1_4/q72/extended.txt | 22 +- .../approved-plans-v1_4/q72/simplified.txt | 22 +- .../approved-plans-v1_4/q73/explain.txt | 20 +- .../approved-plans-v1_4/q73/extended.txt | 10 +- .../approved-plans-v1_4/q73/simplified.txt | 10 +- .../approved-plans-v1_4/q74/explain.txt | 32 +- .../approved-plans-v1_4/q74/extended.txt | 20 +- .../approved-plans-v1_4/q74/simplified.txt | 16 +- .../approved-plans-v1_4/q75/explain.txt | 32 +- .../approved-plans-v1_4/q75/extended.txt | 40 +- .../approved-plans-v1_4/q75/simplified.txt | 16 +- .../approved-plans-v1_4/q76/explain.txt | 26 +- .../approved-plans-v1_4/q76/extended.txt | 18 +- .../approved-plans-v1_4/q76/simplified.txt | 10 +- .../approved-plans-v1_4/q77/explain.txt | 16 +- .../approved-plans-v1_4/q77/extended.txt | 22 +- .../approved-plans-v1_4/q77/simplified.txt | 8 +- .../approved-plans-v1_4/q78/explain.txt | 20 +- .../approved-plans-v1_4/q78/extended.txt | 14 +- .../approved-plans-v1_4/q78/simplified.txt | 10 +- .../approved-plans-v1_4/q79/explain.txt | 20 +- .../approved-plans-v1_4/q79/extended.txt | 10 +- .../approved-plans-v1_4/q79/simplified.txt | 10 +- .../approved-plans-v1_4/q8/explain.txt | 24 +- .../approved-plans-v1_4/q8/extended.txt | 12 +- .../approved-plans-v1_4/q8/simplified.txt | 12 +- .../approved-plans-v1_4/q80/explain.txt | 40 +- .../approved-plans-v1_4/q80/extended.txt | 32 +- .../approved-plans-v1_4/q80/simplified.txt | 20 +- .../approved-plans-v1_4/q81/explain.txt | 20 +- .../approved-plans-v1_4/q81/extended.txt | 14 +- .../approved-plans-v1_4/q81/simplified.txt | 10 +- .../approved-plans-v1_4/q82/explain.txt | 16 +- .../approved-plans-v1_4/q82/extended.txt | 8 +- .../approved-plans-v1_4/q82/simplified.txt | 8 +- .../approved-plans-v1_4/q83/explain.txt | 32 +- .../approved-plans-v1_4/q83/extended.txt | 36 +- .../approved-plans-v1_4/q83/simplified.txt | 16 +- .../approved-plans-v1_4/q84/explain.txt | 24 +- .../approved-plans-v1_4/q84/extended.txt | 12 +- .../approved-plans-v1_4/q84/simplified.txt | 12 +- .../approved-plans-v1_4/q85/explain.txt | 32 +- .../approved-plans-v1_4/q85/extended.txt | 16 +- .../approved-plans-v1_4/q85/simplified.txt | 16 +- .../approved-plans-v1_4/q86/explain.txt | 12 +- .../approved-plans-v1_4/q86/extended.txt | 6 +- .../approved-plans-v1_4/q86/simplified.txt | 6 +- .../approved-plans-v1_4/q87/explain.txt | 12 +- .../approved-plans-v1_4/q87/extended.txt | 14 +- .../approved-plans-v1_4/q87/simplified.txt | 6 +- .../approved-plans-v1_4/q88/explain.txt | 72 +-- .../approved-plans-v1_4/q88/extended.txt | 64 +- .../approved-plans-v1_4/q88/simplified.txt | 36 +- .../approved-plans-v1_4/q89/explain.txt | 16 +- .../approved-plans-v1_4/q89/extended.txt | 8 +- .../approved-plans-v1_4/q89/simplified.txt | 8 +- .../approved-plans-v1_4/q9/explain.txt | 24 +- .../approved-plans-v1_4/q9/extended.txt | 12 +- .../approved-plans-v1_4/q9/simplified.txt | 12 +- .../approved-plans-v1_4/q90/explain.txt | 24 +- .../approved-plans-v1_4/q90/extended.txt | 16 +- .../approved-plans-v1_4/q90/simplified.txt | 12 +- .../approved-plans-v1_4/q91/explain.txt | 28 +- .../approved-plans-v1_4/q91/extended.txt | 14 +- .../approved-plans-v1_4/q91/simplified.txt | 14 +- .../approved-plans-v1_4/q92/explain.txt | 12 +- .../approved-plans-v1_4/q92/extended.txt | 8 +- .../approved-plans-v1_4/q92/simplified.txt | 6 +- .../approved-plans-v1_4/q93/explain.txt | 12 +- .../approved-plans-v1_4/q93/extended.txt | 6 +- .../approved-plans-v1_4/q93/simplified.txt | 6 +- .../approved-plans-v1_4/q94/explain.txt | 24 +- .../approved-plans-v1_4/q94/extended.txt | 12 +- .../approved-plans-v1_4/q94/simplified.txt | 12 +- .../approved-plans-v1_4/q95/explain.txt | 24 +- .../approved-plans-v1_4/q95/extended.txt | 18 +- .../approved-plans-v1_4/q95/simplified.txt | 12 +- .../approved-plans-v1_4/q96/explain.txt | 16 +- .../approved-plans-v1_4/q96/extended.txt | 8 +- .../approved-plans-v1_4/q96/simplified.txt | 8 +- .../approved-plans-v1_4/q97/explain.txt | 8 +- .../approved-plans-v1_4/q97/extended.txt | 6 +- .../approved-plans-v1_4/q97/simplified.txt | 4 +- .../approved-plans-v1_4/q98/explain.txt | 12 +- .../approved-plans-v1_4/q98/extended.txt | 6 +- .../approved-plans-v1_4/q98/simplified.txt | 6 +- .../approved-plans-v1_4/q99/explain.txt | 20 +- .../approved-plans-v1_4/q99/extended.txt | 10 +- .../approved-plans-v1_4/q99/simplified.txt | 10 +- .../approved-plans-v2_7/q10a/explain.txt | 20 +- .../approved-plans-v2_7/q10a/extended.txt | 14 +- .../approved-plans-v2_7/q10a/simplified.txt | 10 +- .../approved-plans-v2_7/q11/explain.txt | 32 +- .../approved-plans-v2_7/q11/extended.txt | 20 +- .../approved-plans-v2_7/q11/simplified.txt | 16 +- .../approved-plans-v2_7/q12/explain.txt | 12 +- .../approved-plans-v2_7/q12/extended.txt | 6 +- .../approved-plans-v2_7/q12/simplified.txt | 6 +- .../approved-plans-v2_7/q14/explain.txt | 48 +- .../approved-plans-v2_7/q14/extended.txt | 86 +-- .../approved-plans-v2_7/q14/simplified.txt | 24 +- .../approved-plans-v2_7/q14a/explain.txt | 40 +- .../approved-plans-v2_7/q14a/extended.txt | 590 +++++++++--------- .../approved-plans-v2_7/q14a/simplified.txt | 20 +- .../approved-plans-v2_7/q18a/explain.txt | 44 +- .../approved-plans-v2_7/q18a/extended.txt | 62 +- .../approved-plans-v2_7/q18a/simplified.txt | 22 +- .../approved-plans-v2_7/q20/explain.txt | 12 +- .../approved-plans-v2_7/q20/extended.txt | 6 +- .../approved-plans-v2_7/q20/simplified.txt | 6 +- .../approved-plans-v2_7/q22/explain.txt | 16 +- .../approved-plans-v2_7/q22/extended.txt | 8 +- .../approved-plans-v2_7/q22/simplified.txt | 8 +- .../approved-plans-v2_7/q22a/explain.txt | 16 +- .../approved-plans-v2_7/q22a/extended.txt | 40 +- .../approved-plans-v2_7/q22a/simplified.txt | 8 +- .../approved-plans-v2_7/q24/explain.txt | 28 +- .../approved-plans-v2_7/q24/extended.txt | 24 +- .../approved-plans-v2_7/q24/simplified.txt | 14 +- .../approved-plans-v2_7/q27a/explain.txt | 28 +- .../approved-plans-v2_7/q27a/extended.txt | 26 +- .../approved-plans-v2_7/q27a/simplified.txt | 14 +- .../approved-plans-v2_7/q34/explain.txt | 20 +- .../approved-plans-v2_7/q34/extended.txt | 10 +- .../approved-plans-v2_7/q34/simplified.txt | 10 +- .../approved-plans-v2_7/q35/explain.txt | 20 +- .../approved-plans-v2_7/q35/extended.txt | 14 +- .../approved-plans-v2_7/q35/simplified.txt | 10 +- .../approved-plans-v2_7/q35a/explain.txt | 20 +- .../approved-plans-v2_7/q35a/extended.txt | 14 +- .../approved-plans-v2_7/q35a/simplified.txt | 10 +- .../approved-plans-v2_7/q36a/explain.txt | 16 +- .../approved-plans-v2_7/q36a/extended.txt | 24 +- .../approved-plans-v2_7/q36a/simplified.txt | 8 +- .../approved-plans-v2_7/q47/explain.txt | 16 +- .../approved-plans-v2_7/q47/extended.txt | 24 +- .../approved-plans-v2_7/q47/simplified.txt | 8 +- .../approved-plans-v2_7/q49/explain.txt | 20 +- .../approved-plans-v2_7/q49/extended.txt | 14 +- .../approved-plans-v2_7/q49/simplified.txt | 10 +- .../approved-plans-v2_7/q51a/explain.txt | 8 +- .../approved-plans-v2_7/q51a/extended.txt | 24 +- .../approved-plans-v2_7/q51a/simplified.txt | 4 +- .../approved-plans-v2_7/q57/explain.txt | 16 +- .../approved-plans-v2_7/q57/extended.txt | 24 +- .../approved-plans-v2_7/q57/simplified.txt | 8 +- .../approved-plans-v2_7/q5a/explain.txt | 24 +- .../approved-plans-v2_7/q5a/extended.txt | 48 +- .../approved-plans-v2_7/q5a/simplified.txt | 12 +- .../approved-plans-v2_7/q6/explain.txt | 28 +- .../approved-plans-v2_7/q6/extended.txt | 14 +- .../approved-plans-v2_7/q6/simplified.txt | 14 +- .../approved-plans-v2_7/q64/explain.txt | 68 +- .../approved-plans-v2_7/q64/extended.txt | 76 +-- .../approved-plans-v2_7/q64/simplified.txt | 34 +- .../approved-plans-v2_7/q67a/explain.txt | 16 +- .../approved-plans-v2_7/q67a/extended.txt | 72 +-- .../approved-plans-v2_7/q67a/simplified.txt | 8 +- .../approved-plans-v2_7/q70a/explain.txt | 16 +- .../approved-plans-v2_7/q70a/extended.txt | 30 +- .../approved-plans-v2_7/q70a/simplified.txt | 8 +- .../approved-plans-v2_7/q72/explain.txt | 46 +- .../approved-plans-v2_7/q72/extended.txt | 22 +- .../approved-plans-v2_7/q72/simplified.txt | 22 +- .../approved-plans-v2_7/q74/explain.txt | 32 +- .../approved-plans-v2_7/q74/extended.txt | 20 +- .../approved-plans-v2_7/q74/simplified.txt | 16 +- .../approved-plans-v2_7/q75/explain.txt | 32 +- .../approved-plans-v2_7/q75/extended.txt | 40 +- .../approved-plans-v2_7/q75/simplified.txt | 16 +- .../approved-plans-v2_7/q77a/explain.txt | 16 +- .../approved-plans-v2_7/q77a/extended.txt | 66 +- .../approved-plans-v2_7/q77a/simplified.txt | 8 +- .../approved-plans-v2_7/q78/explain.txt | 20 +- .../approved-plans-v2_7/q78/extended.txt | 14 +- .../approved-plans-v2_7/q78/simplified.txt | 10 +- .../approved-plans-v2_7/q80a/explain.txt | 40 +- .../approved-plans-v2_7/q80a/extended.txt | 96 +-- .../approved-plans-v2_7/q80a/simplified.txt | 20 +- .../approved-plans-v2_7/q86a/explain.txt | 12 +- .../approved-plans-v2_7/q86a/extended.txt | 18 +- .../approved-plans-v2_7/q86a/simplified.txt | 6 +- .../approved-plans-v2_7/q98/explain.txt | 12 +- .../approved-plans-v2_7/q98/extended.txt | 6 +- .../approved-plans-v2_7/q98/simplified.txt | 6 +- 405 files changed, 4009 insertions(+), 4009 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt index b8ba9b884c..dcaab31388 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt @@ -18,7 +18,7 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (25) : : +- CometFilter (24) : : +- CometHashAggregate (23) @@ -35,11 +35,11 @@ : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + : +- CometNativeScan parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + +- CometNativeScan parquet spark_catalog.default.customer (34) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -54,7 +54,7 @@ ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(29) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#4] Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(36) CometNativeScan parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) + +- CometNativeScan parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(48) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/extended.txt index a6f33d6f7e..ee476d727a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/extended.txt @@ -14,7 +14,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -24,11 +24,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -38,7 +38,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -48,16 +48,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt index e7193f87e1..a95300f752 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -30,11 +30,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -62,7 +62,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt index 091ad62567..a89cd3317e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt @@ -17,13 +17,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -34,13 +34,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (52) : +- CometFilter (51) : +- CometHashAggregate (50) @@ -52,7 +52,7 @@ : : +- CometBroadcastHashJoin (43) : : :- CometProject (39) : : : +- CometFilter (38) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : : : +- CometNativeScan parquet spark_catalog.default.customer (37) : : +- CometBroadcastExchange (42) : : +- CometFilter (41) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) @@ -67,14 +67,14 @@ : +- CometBroadcastHashJoin (61) : :- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : : +- CometNativeScan parquet spark_catalog.default.customer (55) : +- CometBroadcastExchange (60) : +- CometFilter (59) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (58) +- ReusedExchange (63) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -114,7 +114,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -156,7 +156,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -196,7 +196,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -247,7 +247,7 @@ Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight Input [5]: [customer_id#23, year_total#24, customer_id#41, customer_preferred_cust_flag#42, year_total#43] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] -(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(37) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -330,7 +330,7 @@ Arguments: [customer_id#23], [customer_id#66], Inner, BuildRight Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, customer_id#66, year_total#67] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(55) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -422,10 +422,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometFilter (75) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) + +- CometNativeScan parquet spark_catalog.default.date_dim (74) -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(74) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -447,10 +447,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometFilter (79) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (78) + +- CometNativeScan parquet spark_catalog.default.date_dim (78) -(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(78) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/extended.txt index 781eae9054..529dda10fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/extended.txt @@ -16,7 +16,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -24,10 +24,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -38,7 +38,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -46,10 +46,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -61,14 +61,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -79,13 +79,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 80 out of 86 eligible operators (93%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt index 4599ecf539..1e41f9f35e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt @@ -18,7 +18,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -28,10 +28,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -52,10 +52,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] @@ -83,7 +83,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt index 93308784a9..d2fc364869 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject (23) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt index 3f41c97ff5..282edf0ce0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt index fb83fd2f9a..7a0da92e94 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt index 74da8ba883..a85cf5ae8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt @@ -17,22 +17,22 @@ : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) +- CometBroadcastExchange (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) + +- CometNativeScan parquet spark_catalog.default.household_demographics (26) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -47,7 +47,7 @@ ReadSchema: struct= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -71,7 +71,7 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -99,7 +99,7 @@ Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(20) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -155,7 +155,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(26) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#24, hd_dep_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -203,10 +203,10 @@ BroadcastExchange (39) +- * CometColumnarToRow (38) +- CometProject (37) +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) + +- CometNativeScan parquet spark_catalog.default.date_dim (35) -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(35) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/extended.txt index 08e9beb692..a14a409b88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/extended.txt @@ -19,24 +19,24 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : +- CometNativeScan parquet spark_catalog.default.customer_demographics +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + +- CometNativeScan parquet spark_catalog.default.household_demographics Comet accelerated 36 out of 38 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt index 8ef882a435..03a7b35fad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt @@ -23,22 +23,22 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt index 0289abc42d..307a594251 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt @@ -22,7 +22,7 @@ : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -37,7 +37,7 @@ : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) @@ -47,11 +47,11 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -65,12 +65,12 @@ : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) :- CometProject (80) : +- CometFilter (79) : +- CometHashAggregate (78) @@ -115,7 +115,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -138,7 +138,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -161,7 +161,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -185,7 +185,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -328,7 +328,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -360,7 +360,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -684,10 +684,10 @@ BroadcastExchange (124) +- * CometColumnarToRow (123) +- CometProject (122) +- CometFilter (121) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (120) + +- CometNativeScan parquet spark_catalog.default.date_dim (120) -(120) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(120) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -714,10 +714,10 @@ BroadcastExchange (129) +- * CometColumnarToRow (128) +- CometProject (127) +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) + +- CometNativeScan parquet spark_catalog.default.date_dim (125) -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(125) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/extended.txt index 4af04a7846..cbb5339407 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/extended.txt @@ -20,7 +20,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -28,7 +28,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -36,7 +36,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -52,12 +52,12 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -74,11 +74,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -89,15 +89,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -108,20 +108,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -138,11 +138,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -153,15 +153,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -172,15 +172,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometProject : +- CometFilter : : +- ReusedSubquery @@ -199,7 +199,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -216,11 +216,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -231,15 +231,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -250,20 +250,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -280,11 +280,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -295,15 +295,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -314,15 +314,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometProject +- CometFilter : +- ReusedSubquery @@ -341,7 +341,7 @@ CometColumnarToRow : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -358,11 +358,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -373,15 +373,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -392,20 +392,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -422,11 +422,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -437,15 +437,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -456,14 +456,14 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 424 out of 458 eligible operators (92%). Final plan contains 9 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt index 79c782f2ca..a587d19fb5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt @@ -49,12 +49,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -73,11 +73,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -88,11 +88,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 CometProject [i_brand_id,i_class_id,i_category_id] @@ -107,12 +107,12 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #4 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt index bcef4db073..8108f6f7bf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt @@ -17,7 +17,7 @@ : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -32,7 +32,7 @@ : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) @@ -42,11 +42,11 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -60,12 +60,12 @@ : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -83,7 +83,7 @@ +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -121,7 +121,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#60], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(72) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(103) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(108) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) + +- CometNativeScan parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(112) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#99] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) + +- CometNativeScan parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(117) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) + +- CometNativeScan parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(122) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/extended.txt index f56d229b68..5011719f37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -23,7 +23,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -31,7 +31,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -51,13 +51,13 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -74,11 +74,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -89,15 +89,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -108,20 +108,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -138,11 +138,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -153,15 +153,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -172,16 +172,16 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter : +- ReusedSubquery @@ -204,13 +204,13 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -227,11 +227,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -242,15 +242,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -261,20 +261,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -291,11 +291,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -306,15 +306,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -325,15 +325,15 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter : +- ReusedSubquery - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 298 out of 327 eligible operators (91%). Final plan contains 10 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt index fb9abae378..4ae30a0feb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -75,11 +75,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -90,11 +90,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -109,13 +109,13 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt index af9b2efbd1..0003983dd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt @@ -14,15 +14,15 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : : +- CometNativeScan parquet spark_catalog.default.customer (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : +- CometNativeScan parquet spark_catalog.default.customer_address (8) +- CometBroadcastExchange (17) +- CometProject (16) +- CometFilter (15) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#5, c_current_addr_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -61,7 +61,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -89,7 +89,7 @@ Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + +- CometNativeScan parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/extended.txt index 6de0c64850..3089496e4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/extended.txt @@ -16,17 +16,17 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 26 out of 28 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt index c39b96efe3..e1b5409c8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt index f2473f1f01..2a3c8932ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) + +- CometNativeScan parquet spark_catalog.default.call_center (29) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(6) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -92,7 +92,7 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(12) CometNativeScan parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -115,7 +115,7 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(29) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/extended.txt index 2ad029e444..9889eeaca3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/extended.txt @@ -18,26 +18,26 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : +- CometSort : : : +- CometExchange : : : +- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + +- CometNativeScan parquet spark_catalog.default.call_center Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt index 8427aa49a9..429d83d08c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [cs_order_number] #2 CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] CometSort [cs_warehouse_sk,cs_order_number] CometExchange [cs_order_number] #3 CometProject [cs_warehouse_sk,cs_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] CometSort [cr_order_number] CometExchange [cr_order_number] #4 CometProject [cr_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cc_call_center_sk] #7 CometProject [cc_call_center_sk] CometFilter [cc_call_center_sk,cc_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt index f502a5cd8d..faa4f6694b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt @@ -29,20 +29,20 @@ : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + : +- CometNativeScan parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + +- CometNativeScan parquet spark_catalog.default.item (34) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -107,7 +107,7 @@ Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_quarter_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -135,7 +135,7 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_quarter_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -175,7 +175,7 @@ Arguments: [cs_sold_date_sk#17], [d_date_sk#23], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#23] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(28) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_state#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -203,7 +203,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#24, s_state#26] Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(34) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -259,10 +259,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_quarter_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -289,10 +289,10 @@ BroadcastExchange (54) +- * CometColumnarToRow (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) + +- CometNativeScan parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(50) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_quarter_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/extended.txt index 4e1a4f3ec0..39aa7c3908 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/extended.txt @@ -24,7 +24,7 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -33,7 +33,7 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -41,22 +41,22 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 52 out of 57 eligible operators (91%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt index 1d48f96b09..a2b650507d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] @@ -39,7 +39,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] @@ -47,17 +47,17 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] ReusedExchange [d_date_sk] #7 CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt index 5fefd21bfa..7da120375f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt @@ -22,26 +22,26 @@ : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometProject (22) : : +- CometFilter (21) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : +- CometNativeScan parquet spark_catalog.default.date_dim (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + +- CometNativeScan parquet spark_catalog.default.item (32) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -56,7 +56,7 @@ ReadSchema: struct @@ -77,7 +77,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -105,7 +105,7 @@ Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(15) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#17, c_current_addr_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -129,7 +129,7 @@ Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#19, ca_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -157,7 +157,7 @@ Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(26) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#22, s_zip#23] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/extended.txt index a8cfde2d31..fbe6798e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/extended.txt @@ -15,24 +15,24 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 35 out of 35 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt index 93ab89c142..675500cd0b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt @@ -17,22 +17,22 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_zip] #5 CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [s_store_sk,s_zip] #6 CometProject [s_zip] [s_store_sk,s_zip] CometFilter [s_store_sk,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt index e13ae7371e..22fbab2817 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt @@ -13,17 +13,17 @@ : : +- CometBroadcastHashJoin (10) : : :- CometUnion (5) : : : :- CometProject (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometProject (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (3) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (3) : : +- CometBroadcastExchange (9) : : +- CometProject (8) : : +- CometFilter (7) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (6) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (6) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (15) +- CometBroadcastExchange (29) +- CometProject (28) +- CometBroadcastHashJoin (27) @@ -32,13 +32,13 @@ +- CometBroadcastExchange (26) +- CometProject (25) +- CometFilter (24) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (23) + +- CometNativeScan parquet spark_catalog.default.date_dim (23) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#2)] ReadSchema: struct @@ -46,10 +46,10 @@ ReadSchema: struct Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(3) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#6)] ReadSchema: struct @@ -61,7 +61,7 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(6) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#1 Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(15) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -139,7 +139,7 @@ Input [8]: [d_week_seq#10, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#4 Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(23) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/extended.txt index 15f5db847d..7e6f4d3a3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/extended.txt @@ -12,17 +12,17 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometUnion : : : :- CometProject - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin @@ -33,16 +33,16 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometUnion : : :- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometProject - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt index 852c5fca0f..e4b6e81639 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] CometUnion [sold_date_sk,sales_price] CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] @@ -33,4 +33,4 @@ WholeStageCodegen (1) CometBroadcastExchange [d_week_seq] #6 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt index c23383e09a..202b8afcda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject (23) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt index cd52b2cd12..754a4fad60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt index 2958d060fe..7d40087d4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt index 5656916223..d8a4f52824 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt @@ -15,14 +15,14 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + : +- CometNativeScan parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory @@ -37,7 +37,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -115,7 +115,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(10) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -182,7 +182,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(25) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -197,7 +197,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(28) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,7 +256,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -380,10 +380,10 @@ BroadcastExchange (72) +- * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) + +- CometNativeScan parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(68) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,10 +410,10 @@ BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometProject (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -453,7 +453,7 @@ Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquer +- CometBroadcastExchange (86) +- CometProject (85) +- CometFilter (84) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (83) + +- CometNativeScan parquet spark_catalog.default.date_dim (83) (78) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -480,7 +480,7 @@ Arguments: [ss_customer_sk#44], [c_customer_sk#49], Inner, BuildRight Input [5]: [ss_customer_sk#44, ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49] Arguments: [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49], [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49] -(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(83) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#50, d_year#51] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -544,10 +544,10 @@ BroadcastExchange (100) +- * CometColumnarToRow (99) +- CometProject (98) +- CometFilter (97) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (96) + +- CometNativeScan parquet spark_catalog.default.date_dim (96) -(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(96) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#50, d_year#51] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/extended.txt index a84226b077..ad7c278937 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/extended.txt @@ -17,7 +17,7 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter @@ -35,14 +35,14 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometSort : : +- CometProject : : +- CometFilter @@ -65,14 +65,14 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -80,14 +80,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometProject +- CometBroadcastHashJoin :- CometProject @@ -115,14 +115,14 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometSort : +- CometProject : +- CometFilter @@ -134,13 +134,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 127 out of 138 eligible operators (92%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt index 8f1bddf6c0..90a188196a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -41,14 +41,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -75,12 +75,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] @@ -88,14 +88,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_quantity,ws_list_price] [sales] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt index 67e4e39057..4cbe86c1ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt @@ -31,10 +31,10 @@ : : : : : +- CometBroadcastExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (5) : : : : +- CometBroadcastExchange (13) : : : : +- CometFilter (12) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : +- CometSort (39) : : : +- CometProject (38) : : : +- CometFilter (37) @@ -45,17 +45,17 @@ : : : +- CometBroadcastHashJoin (32) : : : :- CometProject (28) : : : : +- CometFilter (27) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) : : : +- CometBroadcastExchange (31) : : : +- CometFilter (30) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + : : : +- CometNativeScan parquet spark_catalog.default.customer (29) : : +- CometBroadcastExchange (52) : : +- CometProject (51) : : +- CometSortMergeJoin (50) : : :- CometSort (44) : : : +- CometExchange (43) : : : +- CometFilter (42) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) + : : : +- CometNativeScan parquet spark_catalog.default.customer (41) : : +- CometSort (49) : : +- CometProject (48) : : +- CometFilter (47) @@ -64,7 +64,7 @@ : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) +- CometHashAggregate (85) +- CometExchange (84) +- CometHashAggregate (83) @@ -113,7 +113,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -141,7 +141,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -208,7 +208,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -223,7 +223,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(29) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -278,7 +278,7 @@ Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold Right output [1]: [c_customer_sk#23] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(41) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -339,7 +339,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#29], Inner, BuildRight Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#29, c_first_name#32, c_last_name#33] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -494,10 +494,10 @@ BroadcastExchange (93) +- * CometColumnarToRow (92) +- CometProject (91) +- CometFilter (90) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (89) + +- CometNativeScan parquet spark_catalog.default.date_dim (89) -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(89) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -524,10 +524,10 @@ BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometProject (96) +- CometFilter (95) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + +- CometNativeScan parquet spark_catalog.default.date_dim (94) -(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(94) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -567,7 +567,7 @@ Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquer +- CometBroadcastExchange (107) +- CometProject (106) +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (104) (99) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -594,7 +594,7 @@ Arguments: [ss_customer_sk#53], [c_customer_sk#58], Inner, BuildRight Input [5]: [ss_customer_sk#53, ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58] Arguments: [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58], [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(104) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#59, d_year#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -658,10 +658,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) + +- CometNativeScan parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(117) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#59, d_year#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/extended.txt index cd91f5de49..47d6a66c8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/extended.txt @@ -20,7 +20,7 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter @@ -38,14 +38,14 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometSort : : : +- CometProject : : : +- CometFilter @@ -68,14 +68,14 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometHashAggregate : : : +- CometExchange : : : +- CometHashAggregate @@ -83,17 +83,17 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometProject : : +- CometSortMergeJoin : : :- CometSort : : : +- CometExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometSort : : +- CometProject : : +- CometFilter @@ -105,14 +105,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -145,14 +145,14 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometSort : : +- CometProject : : +- CometFilter @@ -164,17 +164,17 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometSortMergeJoin : :- CometSort : : +- CometExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometSort : +- CometProject : +- CometFilter @@ -186,13 +186,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 177 out of 190 eligible operators (93%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt index 92563114ae..94b60764fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt @@ -24,7 +24,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -44,14 +44,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -78,12 +78,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] @@ -91,17 +91,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] CometExchange [c_customer_sk] #16 CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -111,7 +111,7 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #17 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] CometExchange [c_last_name,c_first_name] #18 CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt index e478cdfc01..acc9219af4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : : +- CometNativeScan parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + : +- CometNativeScan parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(19) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(32) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : : +- CometNativeScan parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(57) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt index 8777e81a37..b65f56f327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt @@ -23,29 +23,29 @@ Filter : : : : : : +- CometExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : +- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item +: : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +: : +- CometNativeScan parquet spark_catalog.default.customer : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter -: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +: +- CometNativeScan parquet spark_catalog.default.customer_address +- HashAggregate +- CometColumnarToRow +- CometColumnarExchange @@ -69,28 +69,28 @@ Filter : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + +- CometNativeScan parquet spark_catalog.default.customer_address Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt index 440f477d0f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt index 548959be1b..dbebe8f96d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : : +- CometNativeScan parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + : +- CometNativeScan parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(19) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(32) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : : +- CometNativeScan parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(57) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt index 8777e81a37..b65f56f327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt @@ -23,29 +23,29 @@ Filter : : : : : : +- CometExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : +- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item +: : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +: : +- CometNativeScan parquet spark_catalog.default.customer : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter -: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +: +- CometNativeScan parquet spark_catalog.default.customer_address +- HashAggregate +- CometColumnarToRow +- CometColumnarExchange @@ -69,28 +69,28 @@ Filter : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + +- CometNativeScan parquet spark_catalog.default.customer_address Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt index 440f477d0f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt index 6a9f9094d3..4e05ac52eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt @@ -29,20 +29,20 @@ : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + : +- CometNativeScan parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + +- CometNativeScan parquet spark_catalog.default.item (34) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -107,7 +107,7 @@ Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -135,7 +135,7 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#19] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#22, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -175,7 +175,7 @@ Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#25] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(28) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -203,7 +203,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#26], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_sk#26, s_store_id#29, s_store_name#28] Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(34) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -259,10 +259,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -289,10 +289,10 @@ BroadcastExchange (54) +- * CometColumnarToRow (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) + +- CometNativeScan parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(50) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#22, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/extended.txt index 4e1a4f3ec0..39aa7c3908 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/extended.txt @@ -24,7 +24,7 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -33,7 +33,7 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -41,22 +41,22 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 52 out of 57 eligible operators (91%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt index dcad304452..380cb2db83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #3 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] @@ -39,7 +39,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #5 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] @@ -47,17 +47,17 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] ReusedExchange [d_date_sk] #7 CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #8 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt index 3d70460689..180fc4396f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt @@ -17,19 +17,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometNativeScan parquet spark_catalog.default.item (15) +- CometBroadcastExchange (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + +- CometNativeScan parquet spark_catalog.default.promotion (21) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -44,7 +44,7 @@ ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt index 7bdad4b94f..c19e6caf52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt index 44c6fdad28..c4ecdc07fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt @@ -20,11 +20,11 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : : +- CometBroadcastExchange (34) : : +- CometFilter (33) : : +- CometHashAggregate (32) @@ -44,11 +44,11 @@ : +- CometBroadcastExchange (40) : +- CometProject (39) : +- CometFilter (38) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : +- CometNativeScan parquet spark_catalog.default.customer (37) +- CometBroadcastExchange (46) +- CometProject (45) +- CometFilter (44) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns @@ -63,7 +63,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_county#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -177,7 +177,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] Condition : isnotnull(ss_addr_sk#11) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(18) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -248,7 +248,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_addr_sk#23) -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(33) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -513,10 +513,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dyn BroadcastExchange (94) +- * CometColumnarToRow (93) +- CometFilter (92) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (91) + +- CometNativeScan parquet spark_catalog.default.date_dim (91) -(91) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(91) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -538,10 +538,10 @@ Subquery:2 Hosting operator id = 16 Hosting Expression = ss_sold_date_sk#13 IN d BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometFilter (96) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (95) + +- CometNativeScan parquet spark_catalog.default.date_dim (95) -(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(95) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -563,10 +563,10 @@ Subquery:3 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN d BroadcastExchange (102) +- * CometColumnarToRow (101) +- CometFilter (100) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (99) + +- CometNativeScan parquet spark_catalog.default.date_dim (99) -(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(99) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/extended.txt index b4e7aa4db6..4979461c17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/extended.txt @@ -22,13 +22,13 @@ CometColumnarToRow : : : : : : : +- BroadcastExchange : : : : : : : +- CometColumnarToRow : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : : +- CometBroadcastExchange : : : : +- CometHashAggregate : : : : +- CometExchange @@ -43,13 +43,13 @@ CometColumnarToRow : : : : : : +- BroadcastExchange : : : : : : +- CometColumnarToRow : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometHashAggregate : : : +- CometExchange @@ -64,13 +64,13 @@ CometColumnarToRow : : : : : +- BroadcastExchange : : : : : +- CometColumnarToRow : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -84,10 +84,10 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometHashAggregate : +- CometExchange @@ -101,10 +101,10 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -118,9 +118,9 @@ CometColumnarToRow : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + +- CometNativeScan parquet spark_catalog.default.customer_address Comet accelerated 111 out of 120 eligible operators (92%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt index 2f97384320..6d5f88db6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt @@ -26,13 +26,13 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [ca_address_sk,ca_county] #5 CometFilter [ca_address_sk,ca_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #6 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #7 @@ -49,10 +49,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,store_sales] #10 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] @@ -70,10 +70,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #14 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt index b27d40f91b..bac8e60b9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt @@ -14,7 +14,7 @@ : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometNativeScan parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -27,7 +27,7 @@ : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Condition : isnotnull(cs_item_sk#7) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + +- CometNativeScan parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/extended.txt index 943bb147ff..b3c1a352cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/extended.txt @@ -15,11 +15,11 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -33,10 +33,10 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 35 out of 38 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt index 07619fc999..6a8be191c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] @@ -37,5 +37,5 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt index b78621c4bb..5d188a7e09 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt @@ -19,19 +19,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : : +- CometNativeScan parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + : +- CometNativeScan parquet spark_catalog.default.item (17) :- CometHashAggregate (48) : +- CometExchange (47) : +- CometHashAggregate (46) @@ -48,11 +48,11 @@ : +- CometBroadcastExchange (43) : +- CometBroadcastHashJoin (42) : :- CometFilter (37) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (36) + : : +- CometNativeScan parquet spark_catalog.default.item (36) : +- CometBroadcastExchange (41) : +- CometProject (40) : +- CometFilter (39) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (38) + : +- CometNativeScan parquet spark_catalog.default.item (38) +- CometHashAggregate (62) +- CometExchange (61) +- CometHashAggregate (60) @@ -81,7 +81,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_manufact_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_category#13, i_manufact_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,7 +234,7 @@ Arguments: [cs_bill_addr_sk#16], [ca_address_sk#22], Inner, BuildRight Input [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, ca_address_sk#22] Arguments: [cs_item_sk#17, cs_ext_sales_price#18], [cs_item_sk#17, cs_ext_sales_price#18] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(36) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#23, i_manufact_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -245,7 +245,7 @@ ReadSchema: struct Input [2]: [i_item_sk#23, i_manufact_id#24] Condition : isnotnull(i_item_sk#23) -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(38) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_category#13, i_manufact_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -390,10 +390,10 @@ BroadcastExchange (73) +- * CometColumnarToRow (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) + +- CometNativeScan parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(69) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/extended.txt index 063fdb6faf..e1b4b53ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/extended.txt @@ -20,23 +20,23 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -52,19 +52,19 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -80,18 +80,18 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 89 out of 93 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt index ff7856615f..8abe705e18 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt @@ -24,23 +24,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_manufact_id] #6 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #7 CometProject [i_manufact_id] [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_manufact_id] #8 CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] @@ -58,11 +58,11 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_manufact_id] #9 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #10 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_manufact_id] #11 CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt index 287e323952..ce3182de0d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt @@ -19,19 +19,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- CometNativeScan parquet spark_catalog.default.customer (25) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -46,7 +46,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/extended.txt index 3d9d23d2cc..20468961d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/extended.txt @@ -20,22 +20,22 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : +- CometNativeScan parquet spark_catalog.default.household_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 35 out of 37 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt index 47d0e35dd2..6e56af37e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt index f01bb79a29..8595365dcc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt @@ -15,7 +15,7 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) @@ -23,7 +23,7 @@ TakeOrderedAndProject (47) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) @@ -40,15 +40,15 @@ TakeOrderedAndProject (47) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -66,7 +66,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(29) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#20] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18, ca_state#20] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(36) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) + +- CometNativeScan parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(48) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/extended.txt index a6f33d6f7e..ee476d727a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/extended.txt @@ -14,7 +14,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -24,11 +24,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -38,7 +38,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -48,16 +48,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt index f1fe09fb46..5479332ae0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt index ae1d4c242b..c5a32eaa51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt @@ -20,15 +20,15 @@ TakeOrderedAndProject (30) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometNativeScan parquet spark_catalog.default.item (9) +- CometBroadcastExchange (18) +- CometProject (17) +- CometFilter (16) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometNativeScan parquet spark_catalog.default.store (15) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -43,7 +43,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -89,7 +89,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -276,10 +276,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/extended.txt index 55af64efc6..e408ea6536 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/extended.txt @@ -19,15 +19,15 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometHashAggregate : +- CometExchange @@ -42,11 +42,11 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -61,10 +61,10 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 62 out of 66 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt index 7ceccf5d2c..43062e2323 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt @@ -23,15 +23,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometExchange [c_last_name,c_first_name,d_date] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt index ed74780c2c..1b34e155ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt @@ -18,14 +18,14 @@ : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -45,7 +45,7 @@ +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory @@ -60,7 +60,7 @@ ReadSchema: struct 0.000000)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -230,7 +230,7 @@ Arguments: [c_customer_sk#28], [ss_customer_sk#36], Inner, BuildRight Input [14]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -277,7 +277,7 @@ Left output [2]: [customer_id#26, year_total#27] Right output [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] Arguments: [customer_id#26], [customer_id#47], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -360,7 +360,7 @@ Arguments: [customer_id#26], [customer_id#80], Inner, BuildRight Input [12]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, customer_id#80, year_total#81] Arguments: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81], [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -439,7 +439,7 @@ Arguments: [customer_id#26], [customer_id#101], Inner, (CASE WHEN (year_total#81 Input [13]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81, customer_id#101, year_total#102] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(71) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -522,7 +522,7 @@ Arguments: [customer_id#26], [customer_id#128], Inner, BuildRight Input [12]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, customer_id#128, year_total#129] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(89) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -614,10 +614,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dy BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(108) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -639,10 +639,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#41 IN d BroadcastExchange (115) +- * CometColumnarToRow (114) +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) + +- CometNativeScan parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(112) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/extended.txt index 1be0816ee1..c4bd842dd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/extended.txt @@ -19,7 +19,7 @@ CometColumnarToRow : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -27,10 +27,10 @@ CometColumnarToRow : : : : : : +- BroadcastExchange : : : : : : +- CometColumnarToRow : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometHashAggregate : : : : +- CometExchange @@ -41,7 +41,7 @@ CometColumnarToRow : : : : : +- CometBroadcastHashJoin : : : : : :- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -49,10 +49,10 @@ CometColumnarToRow : : : : : +- BroadcastExchange : : : : : +- CometColumnarToRow : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometHashAggregate @@ -64,14 +64,14 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -82,14 +82,14 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -101,14 +101,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -119,13 +119,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 118 out of 126 eligible operators (93%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt index 36be965b48..cd7e079da8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] @@ -31,10 +31,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 @@ -45,7 +45,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] @@ -55,10 +55,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] @@ -70,7 +70,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] @@ -86,7 +86,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] @@ -103,7 +103,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] @@ -119,7 +119,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt index 00e2b8fbc1..6a9f237f6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt @@ -20,18 +20,18 @@ : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (5) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) + : +- CometNativeScan parquet spark_catalog.default.item (18) +- CometBroadcastExchange (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -54,7 +54,7 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(5) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -86,7 +86,7 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(12) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#11, w_state#12] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -114,7 +114,7 @@ Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(18) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -142,7 +142,7 @@ Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -193,10 +193,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dyn BroadcastExchange (37) +- * CometColumnarToRow (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/extended.txt index 54a7b21486..beef88a3fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/extended.txt @@ -19,22 +19,22 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometSort : : : +- CometExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + : : +- CometNativeScan parquet spark_catalog.default.warehouse : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 34 out of 36 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt index 79e3556750..7e7073d139 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt @@ -23,20 +23,20 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] CometExchange [cr_order_number,cr_item_sk] #4 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometBroadcastExchange [w_warehouse_sk,w_state] #5 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #7 CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt index 3b1ad984a3..cbce1b2b45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : +- CometNativeScan parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,10 +17,10 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) + +- CometNativeScan parquet spark_catalog.default.item (4) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -35,7 +35,7 @@ Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_ma Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(4) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/extended.txt index 9d802b5033..b40ebddb6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/extended.txt @@ -7,7 +7,7 @@ CometColumnarToRow +- CometBroadcastHashJoin :- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter @@ -16,6 +16,6 @@ CometColumnarToRow +- CometHashAggregate +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt index 99c935aaa4..2557f51684 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] @@ -18,4 +18,4 @@ WholeStageCodegen (1) CometHashAggregate [i_manufact,count] CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt index 183fffdf00..af01446009 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt index 839937d40c..96ab34454d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 CometProject [i_category] [i_item_sk,i_category_id,i_category] CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt index 62ddd93781..261e6ddbaa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometNativeScan parquet spark_catalog.default.store (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/extended.txt index cb125e2456..5eaea983af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt index 59ad7611a5..c7064b3ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [d_day_name] [d_date_sk,d_day_name] CometFilter [d_date_sk,d_year,d_day_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt index 134ab7939e..e66d71b41e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt @@ -19,7 +19,7 @@ TakeOrderedAndProject (34) : : : +- CometHashAggregate (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- * Sort (21) : : +- * Project (20) : : +- * Filter (19) @@ -31,11 +31,11 @@ TakeOrderedAndProject (34) : +- * CometColumnarToRow (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : +- CometNativeScan parquet spark_catalog.default.item (24) +- ReusedExchange (31) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -131,7 +131,7 @@ Join condition: None Output [3]: [item_sk#7, rnk#11, item_sk#12] Input [4]: [item_sk#7, rnk#11, item_sk#12, rnk#14] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(24) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_product_name#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -189,10 +189,10 @@ Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (38) +- CometProject (37) +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (35) + +- CometNativeScan parquet spark_catalog.default.store_sales (35) -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(35) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt index 5ef0fb3b87..edc1cd324f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt @@ -20,13 +20,13 @@ TakeOrderedAndProject : : : : +- CometHashAggregate : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometHashAggregate : : : +- CometExchange : : : +- CometHashAggregate : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- Sort : : +- Project : : +- Filter @@ -42,22 +42,22 @@ TakeOrderedAndProject : : : +- CometHashAggregate : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 36 out of 55 eligible operators (65%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt index e3ec416397..6ff27fc6c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt @@ -28,13 +28,13 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] CometProject [ss_store_sk,ss_net_profit] CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] CometExchange [ss_item_sk] #2 CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] CometProject [ss_item_sk,ss_net_profit] CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (4) Sort [rnk] @@ -54,6 +54,6 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] InputAdapter CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt index 9d3cf3d382..22d421ed40 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt @@ -20,24 +20,24 @@ TakeOrderedAndProject (39) : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + : +- CometNativeScan parquet spark_catalog.default.item (20) +- BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) + +- CometNativeScan parquet spark_catalog.default.item (27) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -52,7 +52,7 @@ ReadSchema: struct= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -66,7 +66,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(8) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -94,7 +94,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(14) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -122,7 +122,7 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH, Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(20) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -174,10 +174,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/extended.txt index 57fbc2d035..fe50450ec6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/extended.txt @@ -17,21 +17,21 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 31 out of 33 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt index e29e101c7e..0f5cdba34b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt @@ -21,19 +21,19 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt index 338f5f4201..260bd1c80b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt @@ -26,11 +26,11 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometProject (7) : : +- CometFilter (6) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) :- * Project (49) : +- * Filter (48) : +- Window (47) @@ -52,7 +52,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) : : +- CometProject (33) : : +- CometFilter (32) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) : +- ReusedExchange (36) +- * Project (72) +- * Filter (71) @@ -75,7 +75,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) : +- CometProject (56) : +- CometFilter (55) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) +- ReusedExchange (59) @@ -99,7 +99,7 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(5) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -123,7 +123,7 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(10) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -216,7 +216,7 @@ Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, c Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(31) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -317,7 +317,7 @@ Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(54) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -430,10 +430,10 @@ BroadcastExchange (83) +- * CometColumnarToRow (82) +- CometProject (81) +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) + +- CometNativeScan parquet spark_catalog.default.date_dim (79) -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(79) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt index 75684a9669..e8140d659e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt @@ -28,14 +28,14 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : +- CometNativeScan parquet spark_catalog.default.web_returns : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- Project : +- Filter : +- Window @@ -58,11 +58,11 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- Project +- Filter +- Window @@ -85,10 +85,10 @@ CometColumnarToRow : : +- ReusedSubquery : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : +- CometNativeScan parquet spark_catalog.default.store_returns +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 66 out of 87 eligible operators (75%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt index ca80833ee5..d91eb10d33 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt @@ -40,14 +40,14 @@ WholeStageCodegen (11) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] WholeStageCodegen (6) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -76,7 +76,7 @@ WholeStageCodegen (11) ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] @@ -106,5 +106,5 @@ WholeStageCodegen (11) ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt index b4a1dedd61..7dfc78495f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt @@ -23,11 +23,11 @@ : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- CometNativeScan parquet spark_catalog.default.store (14) :- CometHashAggregate (41) : +- CometExchange (40) : +- CometHashAggregate (39) @@ -46,7 +46,7 @@ : +- CometBroadcastExchange (36) : +- CometProject (35) : +- CometFilter (34) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (33) + : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) +- CometHashAggregate (64) +- CometExchange (63) +- CometHashAggregate (62) @@ -64,12 +64,12 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (45) : : +- CometProject (49) : : +- CometFilter (48) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) : +- ReusedExchange (53) +- CometBroadcastExchange (59) +- CometProject (58) +- CometFilter (57) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (56) + +- CometNativeScan parquet spark_catalog.default.web_site (56) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -108,7 +108,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,7 +136,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -226,7 +226,7 @@ Arguments: [date_sk#37], [d_date_sk#52], Inner, BuildRight Input [7]: [page_sk#36, date_sk#37, sales_price#38, profit#39, return_amt#40, net_loss#41, d_date_sk#52] Arguments: [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41], [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41] -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +(33) CometNativeScan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -295,7 +295,7 @@ ReadSchema: struct @@ -67,7 +67,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_numb Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(8) CometNativeScan parquet spark_catalog.default.store Output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -95,7 +95,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [1]: [d_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -119,7 +119,7 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#26], Inner, BuildRight Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#26] Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -175,10 +175,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/extended.txt index 0fbba7d1e8..935b0a2e1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/extended.txt @@ -12,7 +12,7 @@ CometColumnarToRow : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -21,17 +21,17 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 31 out of 33 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt index 98a44eaf40..5eddc939fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] @@ -25,15 +25,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #4 CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastExchange [d_date_sk] #5 CometFilter [d_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt index 683cc25d24..be8e4fd945 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt @@ -24,7 +24,7 @@ TakeOrderedAndProject (41) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) +- CometSort (33) +- CometColumnarExchange (32) +- * Project (31) @@ -54,7 +54,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -227,10 +227,10 @@ BroadcastExchange (46) +- * CometColumnarToRow (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) + +- CometNativeScan parquet spark_catalog.default.date_dim (42) -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(42) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt index 66c5717cc7..2465344e99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt @@ -25,11 +25,11 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometSort +- CometColumnarExchange +- Project @@ -48,6 +48,6 @@ TakeOrderedAndProject +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 37 out of 47 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt index a5af2e114b..b8034c5f59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt @@ -35,11 +35,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] CometColumnarExchange [item_sk,d_date] #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt index a05651a422..4af1b4f562 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt index 218062c3ef..cee223d027 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt index 2fc38bf838..d77e8c3b09 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometNativeScan parquet spark_catalog.default.store (15) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt index 825b1ed81c..77b01e59ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -25,13 +25,13 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 27 out of 33 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt index 0c123eb728..827a521610 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manufact_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] @@ -31,11 +31,11 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [d_date_sk,d_qoy] #5 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [s_store_sk] #6 CometFilter [s_store_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt index 247d1d2d5a..5e2af901c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt @@ -34,29 +34,29 @@ : : : : : : +- CometBroadcastExchange (11) : : : : : : +- CometProject (10) : : : : : : +- CometFilter (9) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (8) : : : : : +- CometBroadcastExchange (17) : : : : : +- CometProject (16) : : : : : +- CometFilter (15) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) : : : : +- CometBroadcastExchange (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (30) : : : +- CometFilter (29) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) : : +- CometBroadcastExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (33) : +- CometBroadcastExchange (42) : +- CometProject (41) : +- CometFilter (40) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + : +- CometNativeScan parquet spark_catalog.default.store (39) +- CometBroadcastExchange (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -95,7 +95,7 @@ Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(8) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_class#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -123,7 +123,7 @@ Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] Arguments: [customer_sk#6], [customer_sk#6] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(20) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#20, c_current_addr_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -214,7 +214,7 @@ Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(33) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -242,7 +242,7 @@ Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(39) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_county#30, s_state#31] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -270,7 +270,7 @@ Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildR Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -340,10 +340,10 @@ BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometProject (61) +- CometFilter (60) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) + +- CometNativeScan parquet spark_catalog.default.date_dim (59) -(59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(59) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -372,10 +372,10 @@ BroadcastExchange (68) +- * CometColumnarToRow (67) +- CometProject (66) +- CometFilter (65) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) + +- CometNativeScan parquet spark_catalog.default.date_dim (64) -(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(64) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -404,10 +404,10 @@ Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) + +- CometNativeScan parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(69) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#44, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -446,10 +446,10 @@ Subquery:5 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (79) +- CometProject (78) +- CometFilter (77) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) + +- CometNativeScan parquet spark_catalog.default.date_dim (76) -(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(76) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#44, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/extended.txt index 98fce92fb1..87af0bb5d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/extended.txt @@ -32,7 +32,7 @@ CometColumnarToRow : : : : : : : : +- CometColumnarToRow : : : : : : : : +- CometProject : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : +- CometProject : : : : : : : +- CometFilter : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -40,14 +40,14 @@ CometColumnarToRow : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -63,7 +63,7 @@ CometColumnarToRow : : : : : +- CometHashAggregate : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- Subquery : : : : +- CometColumnarToRow : : : : +- CometHashAggregate @@ -71,21 +71,21 @@ CometColumnarToRow : : : : +- CometHashAggregate : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter : :- ReusedSubquery : +- ReusedSubquery - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 75 out of 84 eligible operators (89%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt index ae0cecb1ba..4f3529bb52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt @@ -36,7 +36,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] @@ -44,14 +44,14 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk] #5 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] @@ -71,7 +71,7 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #4 WholeStageCodegen (1) CometColumnarToRow @@ -81,19 +81,19 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] CometBroadcastExchange [s_county,s_state] #13 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_county,s_state] CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [(d_month_seq + 1)] #3 ReusedSubquery [(d_month_seq + 3)] #4 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt index 47e0117481..358eb94c69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt index dd7658cbac..a2adacc531 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt index 5562e86b91..a0c0ffb8dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt @@ -19,20 +19,20 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : : +- CometNativeScan parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + : +- CometNativeScan parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_color#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) + +- CometNativeScan parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(63) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/extended.txt index 36b9a8306d..0236f37402 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/extended.txt @@ -20,24 +20,24 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -53,20 +53,20 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -82,19 +82,19 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 92 out of 96 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt index 19eca782cd..350da49c9f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt index eb21a3abd7..df441af4f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(14) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt index de7d26cd06..3ab8670a2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -30,13 +30,13 @@ TakeOrderedAndProject : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : : +- CometNativeScan parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -54,7 +54,7 @@ TakeOrderedAndProject : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -62,13 +62,13 @@ TakeOrderedAndProject : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : +- CometNativeScan parquet spark_catalog.default.call_center +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -86,7 +86,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -94,12 +94,12 @@ TakeOrderedAndProject : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + +- CometNativeScan parquet spark_catalog.default.call_center Comet accelerated 75 out of 97 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt index b854e818db..3bfe35a7e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] @@ -38,13 +38,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt index 85c8622ee4..54527e00f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt @@ -18,12 +18,12 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- CometBroadcastExchange (38) : +- CometFilter (37) @@ -41,7 +41,7 @@ : +- CometProject (30) : +- CometBroadcastHashJoin (29) : :- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- ReusedExchange (28) +- CometBroadcastExchange (53) +- CometFilter (52) @@ -70,7 +70,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -176,7 +176,7 @@ Arguments: [cs_item_sk#14], [i_item_sk#18], Inner, BuildRight Input [5]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_sk#18, i_item_id#19] Arguments: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19], [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_date#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -325,14 +325,14 @@ BroadcastExchange (67) +- CometProject (65) +- CometBroadcastHashJoin (64) :- CometFilter (59) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) + : +- CometNativeScan parquet spark_catalog.default.date_dim (58) +- CometBroadcastExchange (63) +- CometProject (62) +- CometFilter (61) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (60) + +- CometNativeScan parquet spark_catalog.default.date_dim (60) -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(58) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -343,7 +343,7 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(60) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -382,10 +382,10 @@ Subquery:2 Hosting operator id = 61 Hosting Expression = Subquery scalar-subquer * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) + +- CometNativeScan parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(68) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -409,14 +409,14 @@ BroadcastExchange (81) +- CometProject (79) +- CometBroadcastHashJoin (78) :- CometFilter (73) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + : +- CometNativeScan parquet spark_catalog.default.date_dim (72) +- CometBroadcastExchange (77) +- CometProject (76) +- CometFilter (75) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) + +- CometNativeScan parquet spark_catalog.default.date_dim (74) -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(72) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_date#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -427,7 +427,7 @@ ReadSchema: struct Input [2]: [d_date_sk#20, d_date#21] Condition : isnotnull(d_date_sk#20) -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(74) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/extended.txt index c2dd549ac6..e2a1cab1e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/extended.txt @@ -20,7 +20,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter @@ -28,17 +28,17 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter @@ -46,8 +46,8 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -65,26 +65,26 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter : : : : +- ReusedSubquery - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -100,16 +100,16 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter : +- ReusedSubquery - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 98 out of 108 eligible operators (90%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt index 2e709c1b54..14591667b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt @@ -24,7 +24,7 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] [d_date] CometFilter [d_date,d_week_seq] @@ -34,17 +34,17 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cs_item_rev] #6 CometFilter [item_id,cs_item_rev] @@ -65,18 +65,18 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #9 CometProject [d_date] CometFilter [d_date,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] ReusedExchange [i_item_sk,i_item_id] #4 CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #9 CometBroadcastExchange [item_id,ws_item_rev] #11 CometFilter [item_id,ws_item_rev] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt index 6d7e26d609..d25b1ea057 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt @@ -13,19 +13,19 @@ : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (12) + : : +- CometNativeScan parquet spark_catalog.default.store (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) + : +- CometNativeScan parquet spark_catalog.default.date_dim (18) +- CometBroadcastExchange (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -36,17 +36,17 @@ : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) + : +- CometNativeScan parquet spark_catalog.default.store (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -55,7 +55,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -97,7 +97,7 @@ Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, s Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(12) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -125,7 +125,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(18) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -161,7 +161,7 @@ Input [9]: [d_week_seq#5, ss_store_sk#1, sum#38, sum#39, sum#40, sum#41, sum#42, Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(26) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#45, s_store_id#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -189,7 +189,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#45], Inner, BuildRight Input [11]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#45, s_store_id#47] Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(32) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#48, d_week_seq#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/extended.txt index 3d50a2131e..bfd467b72b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/extended.txt @@ -12,19 +12,19 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin @@ -36,18 +36,18 @@ CometColumnarToRow : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt index bca9c6463d..72823b14c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] @@ -37,8 +37,8 @@ WholeStageCodegen (1) CometBroadcastExchange [s_store_sk,s_store_id] #6 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [d_week_seq] #7 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt index b0905506ee..185e47bce6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt @@ -15,22 +15,22 @@ : : : +- CometBroadcastHashJoin (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (1) : : : +- CometBroadcastExchange (6) : : : +- CometFilter (5) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : : +- CometNativeScan parquet spark_catalog.default.customer (4) : : +- CometBroadcastExchange (11) : : +- CometFilter (10) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) +- CometBroadcastExchange (32) +- CometProject (31) +- CometBroadcastHashJoin (30) :- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + : +- CometNativeScan parquet spark_catalog.default.item (20) +- CometBroadcastExchange (29) +- CometFilter (28) +- CometHashAggregate (27) @@ -38,10 +38,10 @@ +- CometHashAggregate (25) +- CometProject (24) +- CometFilter (23) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + +- CometNativeScan parquet spark_catalog.default.item (22) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(1) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -56,7 +56,7 @@ Condition : isnotnull(ca_address_sk#1) Input [2]: [ca_address_sk#1, ca_state#2] Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(4) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -105,7 +105,7 @@ Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -133,7 +133,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(20) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(22) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) + +- CometNativeScan parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(46) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#11, d_year#24, d_moy#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/extended.txt index f5b69fc6cf..5cce29dd89 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/extended.txt @@ -14,10 +14,10 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -33,18 +33,18 @@ CometColumnarToRow : : : +- CometHashAggregate : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -52,6 +52,6 @@ CometColumnarToRow +- CometHashAggregate +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt index 2c772b5998..7318a7bfe7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt @@ -16,10 +16,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt index e5e165b5ec..535ea603da 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt @@ -19,20 +19,20 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : : +- CometNativeScan parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + : +- CometNativeScan parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_category#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) + +- CometNativeScan parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(63) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/extended.txt index 36b9a8306d..0236f37402 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/extended.txt @@ -20,24 +20,24 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -53,20 +53,20 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -82,19 +82,19 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 92 out of 96 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt index 086da3b776..45ea4b263e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt index 4f090653c8..b1f35c55a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt @@ -22,26 +22,26 @@ : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer (21) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + : +- CometNativeScan parquet spark_catalog.default.item (32) +- BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometHashAggregate (61) @@ -78,7 +78,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -123,7 +123,7 @@ Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(18) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/extended.txt index 1fc09bd2e3..6825c99831 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/extended.txt @@ -7,7 +7,7 @@ CometColumnarToRow : :- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometFilter : : +- CometHashAggregate @@ -22,15 +22,15 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -47,6 +47,6 @@ CometColumnarToRow +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 45 out of 48 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt index cd1c84b6f2..f303607d4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] CometFilter [s_store_sk,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] CometHashAggregate [sum] [ss_store_sk,ss_item_sk,revenue,sum(UnscaledValue(ss_sales_price))] @@ -26,15 +26,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #5 CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometBroadcastExchange [ss_store_sk,ave] #6 CometFilter [ss_store_sk,ave] CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt index 7599e9ecd3..d6ed31b8e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt @@ -21,18 +21,18 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) + : +- CometNativeScan parquet spark_catalog.default.ship_mode (20) +- CometHashAggregate (45) +- CometExchange (44) +- CometHashAggregate (43) @@ -64,7 +64,7 @@ ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -173,7 +173,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(28) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -205,7 +205,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#2] Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16] -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(35) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -269,10 +269,10 @@ BroadcastExchange (51) +- * CometColumnarToRow (50) +- CometProject (49) +- CometFilter (48) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + +- CometNativeScan parquet spark_catalog.default.date_dim (47) -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(47) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/extended.txt index b4fbb455ef..738d55c552 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/extended.txt @@ -13,7 +13,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -23,11 +23,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -37,7 +37,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -47,16 +47,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 53 eligible operators (66%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt index 68bf32c40b..56afb9227a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt @@ -17,7 +17,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -29,11 +29,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -61,7 +61,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -69,4 +69,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt index 0e32283618..651a473d07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt @@ -17,19 +17,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometNativeScan parquet spark_catalog.default.item (15) +- CometBroadcastExchange (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + +- CometNativeScan parquet spark_catalog.default.promotion (21) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -44,7 +44,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -92,7 +92,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -118,7 +118,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#7, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -274,10 +274,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt index 6a918ac127..4ba9ebc0fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt @@ -21,17 +21,17 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- BroadcastHashJoin :- CometColumnarToRow : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- BroadcastExchange +- Project +- Filter @@ -51,10 +51,10 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 34 out of 52 eligible operators (65%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt index d6ba61a844..3308d3e39a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt @@ -31,11 +31,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -44,7 +44,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #6 WholeStageCodegen (2) @@ -69,5 +69,5 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt index bb217d6417..3514998a69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt @@ -12,7 +12,7 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometNativeScan parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) @@ -21,7 +21,7 @@ : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) @@ -35,10 +35,10 @@ +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) + +- CometNativeScan parquet spark_catalog.default.time_dim (26) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -69,7 +69,7 @@ ReadSchema: struct @@ -102,7 +102,7 @@ Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < c Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -126,7 +126,7 @@ Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#16, i_item_desc#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -150,7 +150,7 @@ Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#18, cd_marital_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -178,7 +178,7 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#20, hd_buy_potential#21] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -206,7 +206,7 @@ Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -234,7 +234,7 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -258,7 +258,7 @@ Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#28, d_date#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -282,7 +282,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) + +- CometNativeScan parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(65) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/extended.txt index 9000d27d1b..4f2941b209 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/extended.txt @@ -32,41 +32,41 @@ CometColumnarToRow : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : +- CometProject : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion +- CometSort +- CometExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + +- CometNativeScan parquet spark_catalog.default.catalog_returns Comet accelerated 66 out of 68 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt index 678f8c67df..e7af25252a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt index 346e110371..6c24729ac4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt @@ -19,19 +19,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- CometNativeScan parquet spark_catalog.default.customer (25) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -46,7 +46,7 @@ ReadSchema: struct= 1) AND (cnt#17 <= 5)) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/extended.txt index 3d9d23d2cc..20468961d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/extended.txt @@ -20,22 +20,22 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : +- CometNativeScan parquet spark_catalog.default.household_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 35 out of 37 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt index 227dd00f2b..1560bcd1ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt index 649b5dc4d5..0e75a85787 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,7 +51,7 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/extended.txt index 476c7be954..18c1c90d20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -23,10 +23,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -37,7 +37,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -45,10 +45,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -60,14 +60,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -78,13 +78,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 79 out of 85 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt index 72e39422e0..6411ab4b43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -27,10 +27,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -51,10 +51,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] @@ -66,7 +66,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index 7afb1e3be6..e3a8208f70 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -26,15 +26,15 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -51,7 +51,7 @@ : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -68,7 +68,7 @@ : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -92,7 +92,7 @@ : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -137,7 +137,7 @@ ReadSchema: struct @@ -48,7 +48,7 @@ ReadSchema: struct @@ -136,10 +136,10 @@ Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(22) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#29)] PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/extended.txt index b56a6590db..cbb627c528 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/extended.txt @@ -9,39 +9,39 @@ CometColumnarToRow : :- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometProject : +- CometBroadcastHashJoin : :- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometProject +- CometBroadcastHashJoin :- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 44 out of 44 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt index b5449831cb..abb482c3d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt @@ -11,20 +11,20 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category] #2 CometProject [i_category] [i_item_sk,i_category] CometFilter [i_item_sk,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] @@ -32,6 +32,6 @@ WholeStageCodegen (1) CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt index bb7ed0a50c..0a9506ad22 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt @@ -21,10 +21,10 @@ TakeOrderedAndProject (85) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (28) : +- CometHashAggregate (27) : +- CometExchange (26) @@ -71,7 +71,7 @@ TakeOrderedAndProject (85) : : +- ReusedExchange (53) : +- CometBroadcastExchange (58) : +- CometFilter (57) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (56) + : +- CometNativeScan parquet spark_catalog.default.web_page (56) +- CometBroadcastExchange (75) +- CometHashAggregate (74) +- CometExchange (73) @@ -98,7 +98,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -85,7 +85,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -113,7 +113,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(15) CometNativeScan parquet spark_catalog.default.customer_address Output [1]: [ca_zip#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -127,7 +127,7 @@ Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVa Input [1]: [ca_zip#12] Arguments: [ca_zip#13], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) AS ca_zip#13] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(18) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#14, ca_zip#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -142,7 +142,7 @@ Condition : isnotnull(ca_address_sk#14) Input [2]: [ca_address_sk#14, ca_zip#15] Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#15, 10, true, false, true) AS ca_zip#16] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(21) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,10 +256,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/extended.txt index 8e04bbeba7..74d9e64d61 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/extended.txt @@ -16,15 +16,15 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -32,7 +32,7 @@ CometColumnarToRow +- CometBroadcastHashJoin :- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter @@ -43,10 +43,10 @@ CometColumnarToRow +- CometBroadcastHashJoin :- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 46 out of 48 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt index d1542ab607..f20d472e8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometExchange [ca_zip] #6 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_zip] CometBroadcastExchange [ca_zip] #7 CometProject [ca_zip] CometFilter [ca_zip,cnt] @@ -47,8 +47,8 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [c_current_addr_sk] #9 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt index f0a163bf69..c3a57562e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt @@ -27,23 +27,23 @@ : : : : : +- CometExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) : : : : +- CometBroadcastExchange (15) : : : : +- CometProject (14) : : : : +- CometFilter (13) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) : : : +- CometBroadcastExchange (21) : : : +- CometProject (20) : : : +- CometFilter (19) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : : +- CometNativeScan parquet spark_catalog.default.store (18) : : +- CometBroadcastExchange (27) : : +- CometProject (26) : : +- CometFilter (25) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : : +- CometNativeScan parquet spark_catalog.default.item (24) : +- CometBroadcastExchange (33) : +- CometProject (32) : +- CometFilter (31) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + : +- CometNativeScan parquet spark_catalog.default.promotion (30) :- CometHashAggregate (67) : +- CometExchange (66) : +- CometHashAggregate (65) @@ -65,12 +65,12 @@ : : : : : +- CometExchange (46) : : : : : +- CometProject (45) : : : : : +- CometFilter (44) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (43) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (43) : : : : +- ReusedExchange (50) : : : +- CometBroadcastExchange (56) : : : +- CometProject (55) : : : +- CometFilter (54) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (53) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (53) : : +- ReusedExchange (59) : +- ReusedExchange (62) +- CometHashAggregate (96) @@ -94,12 +94,12 @@ : : : : +- CometExchange (75) : : : : +- CometProject (74) : : : : +- CometFilter (73) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (72) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (72) : : : +- ReusedExchange (79) : : +- CometBroadcastExchange (85) : : +- CometProject (84) : : +- CometFilter (83) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (82) + : : +- CometNativeScan parquet spark_catalog.default.web_site (82) : +- ReusedExchange (88) +- ReusedExchange (91) @@ -124,7 +124,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -156,7 +156,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#1 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(12) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -184,7 +184,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(18) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#16, s_store_id#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -212,7 +212,7 @@ Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(24) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_current_price#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -240,7 +240,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(30) CometNativeScan parquet spark_catalog.default.promotion Output [2]: [p_promo_sk#21, p_channel_tv#22] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -302,7 +302,7 @@ Arguments: hashpartitioning(cs_item_sk#29, cs_order_number#31, 5), ENSURE_REQUIR Input [7]: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_order_number#31, cs_ext_sales_price#32, cs_net_profit#33, cs_sold_date_sk#34] Arguments: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_order_number#31, cs_ext_sales_price#32, cs_net_profit#33, cs_sold_date_sk#34], [cs_item_sk#29 ASC NULLS FIRST, cs_order_number#31 ASC NULLS FIRST] -(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(43) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#36, cr_order_number#37, cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -346,7 +346,7 @@ Arguments: [cs_sold_date_sk#34], [d_date_sk#41], Inner, BuildRight Input [9]: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_ext_sales_price#32, cs_net_profit#33, cs_sold_date_sk#34, cr_return_amount#38, cr_net_loss#39, d_date_sk#41] Arguments: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_ext_sales_price#32, cs_net_profit#33, cr_return_amount#38, cr_net_loss#39], [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_ext_sales_price#32, cs_net_profit#33, cr_return_amount#38, cr_net_loss#39] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +(53) CometNativeScan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#42, cp_catalog_page_id#43] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -432,7 +432,7 @@ Arguments: hashpartitioning(ws_item_sk#52, ws_order_number#55, 5), ENSURE_REQUIR Input [7]: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_order_number#55, ws_ext_sales_price#56, ws_net_profit#57, ws_sold_date_sk#58] Arguments: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_order_number#55, ws_ext_sales_price#56, ws_net_profit#57, ws_sold_date_sk#58], [ws_item_sk#52 ASC NULLS FIRST, ws_order_number#55 ASC NULLS FIRST] -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(72) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#60, wr_order_number#61, wr_return_amt#62, wr_net_loss#63, wr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -476,7 +476,7 @@ Arguments: [ws_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight Input [9]: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_ext_sales_price#56, ws_net_profit#57, ws_sold_date_sk#58, wr_return_amt#62, wr_net_loss#63, d_date_sk#65] Arguments: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_ext_sales_price#56, ws_net_profit#57, wr_return_amt#62, wr_net_loss#63], [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_ext_sales_price#56, ws_net_profit#57, wr_return_amt#62, wr_net_loss#63] -(82) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +(82) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#66, web_site_id#67] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] @@ -579,10 +579,10 @@ BroadcastExchange (108) +- * CometColumnarToRow (107) +- CometProject (106) +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (104) -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(104) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/extended.txt index ae6fc19b60..db569f1bf2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/extended.txt @@ -27,28 +27,28 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -71,23 +71,23 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -110,22 +110,22 @@ CometColumnarToRow : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site + : : +- CometNativeScan parquet spark_catalog.default.web_site : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + +- CometNativeScan parquet spark_catalog.default.promotion Comet accelerated 123 out of 127 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt index 198646e0fd..2598a261c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt @@ -31,28 +31,28 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),cp_catalog_page_id] CometExchange [cp_catalog_page_id] #10 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] @@ -75,12 +75,12 @@ WholeStageCodegen (1) CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),web_site_id] @@ -105,11 +105,11 @@ WholeStageCodegen (1) CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt index 8b66e63cf3..715c83feaa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt @@ -20,11 +20,11 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : : +- CometBroadcastExchange (34) : : +- CometFilter (33) : : +- CometHashAggregate (32) @@ -44,11 +44,11 @@ : +- CometBroadcastExchange (40) : +- CometProject (39) : +- CometFilter (38) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : +- CometNativeScan parquet spark_catalog.default.customer (37) +- CometBroadcastExchange (46) +- CometProject (45) +- CometFilter (44) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns @@ -63,7 +63,7 @@ ReadSchema: struct Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -95,7 +95,7 @@ Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -169,7 +169,7 @@ Arguments: [cr_item_sk#12], [i_item_sk#16], Inner, BuildRight Input [5]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14, i_item_sk#16, i_item_id#17] Arguments: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17], [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -310,18 +310,18 @@ BroadcastExchange (68) +- CometProject (66) +- CometBroadcastHashJoin (65) :- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (64) +- CometProject (63) +- CometBroadcastHashJoin (62) - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (57) + :- CometNativeScan parquet spark_catalog.default.date_dim (57) +- CometBroadcastExchange (61) +- CometProject (60) +- CometFilter (59) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) + +- CometNativeScan parquet spark_catalog.default.date_dim (58) -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -332,13 +332,13 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(57) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(58) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -391,15 +391,15 @@ BroadcastExchange (79) +- CometProject (77) +- CometBroadcastHashJoin (76) :- CometFilter (70) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) + : +- CometNativeScan parquet spark_catalog.default.date_dim (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometBroadcastHashJoin (73) - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (71) + :- CometNativeScan parquet spark_catalog.default.date_dim (71) +- ReusedExchange (72) -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(69) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,7 +410,7 @@ ReadSchema: struct Input [2]: [d_date_sk#18, d_date#19] Condition : isnotnull(d_date_sk#18) -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(71) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/extended.txt index d325825292..47584422b2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/extended.txt @@ -19,32 +19,32 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin - : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : :- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin - : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : :- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometHashAggregate : +- CometExchange @@ -61,32 +61,32 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin - : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : :- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin - : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : :- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -101,19 +101,19 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + :- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 109 out of 114 eligible operators (95%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt index bff9c86587..6a07e8cf51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt @@ -23,24 +23,24 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cr_item_qty] #7 CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] @@ -60,18 +60,18 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #10 CometProject [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] ReusedExchange [d_week_seq] #4 ReusedExchange [i_item_sk,i_item_id] #5 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #10 CometBroadcastExchange [item_id,wr_item_qty] #12 CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt index 5bafdc5586..63140cfc2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt @@ -14,27 +14,27 @@ : : : : +- CometBroadcastHashJoin (8) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (4) : : : +- CometBroadcastExchange (12) : : : +- CometFilter (11) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (10) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) + : +- CometNativeScan parquet spark_catalog.default.income_band (20) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) + +- CometNativeScan parquet spark_catalog.default.store_returns (27) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -49,7 +49,7 @@ Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2) Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#1, 16, true, false, true) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#5, 20, true, false, true) AS c_first_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#6, 30, true, false, true) AS c_last_name#9] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(4) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#10, ca_city#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -77,7 +77,7 @@ Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(10) CometNativeScan parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -101,7 +101,7 @@ Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -125,7 +125,7 @@ Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +(20) CometNativeScan parquet spark_catalog.default.income_band Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -157,7 +157,7 @@ Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_c Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(27) CometNativeScan parquet spark_catalog.default.store_returns Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/extended.txt index 5fce933878..b025c868d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/extended.txt @@ -13,23 +13,23 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : +- CometNativeScan parquet spark_catalog.default.income_band +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + +- CometNativeScan parquet spark_catalog.default.store_returns Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt index e43557c27d..9df82b0e98 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt @@ -15,21 +15,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt index c263f03d76..723a82c145 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt @@ -23,30 +23,30 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometProject (16) : : : : +- CometFilter (15) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (14) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + : +- CometNativeScan parquet spark_catalog.default.date_dim (32) +- CometBroadcastExchange (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) + +- CometNativeScan parquet spark_catalog.default.reason (38) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -65,7 +65,7 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(4) CometNativeScan parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -89,7 +89,7 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +(9) CometNativeScan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -113,7 +113,7 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(14) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -141,7 +141,7 @@ Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_s Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(20) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -169,7 +169,7 @@ Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status# Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(26) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -197,7 +197,7 @@ Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(32) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -225,7 +225,7 @@ Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +(38) CometNativeScan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#35, r_reason_desc#36] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -281,10 +281,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/extended.txt index e2383394d1..9d9ce62ec2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/extended.txt @@ -25,32 +25,32 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason + +- CometNativeScan parquet spark_catalog.default.reason Comet accelerated 50 out of 52 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt index dfcca32f03..071d1eba87 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt @@ -29,30 +29,30 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometBroadcastExchange [wp_web_page_sk] #4 CometFilter [wp_web_page_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #7 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt index 1af5e2612c..fb0df08038 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject (24) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -65,7 +65,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -141,10 +141,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + +- CometNativeScan parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt index dfc810b108..88ef4a1db5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt @@ -19,14 +19,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 23 out of 28 eligible operators (82%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt index 29e2d72920..33e882bb0d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt @@ -25,12 +25,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt index fc6b12b834..3a671be2d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt @@ -19,11 +19,11 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) : +- BroadcastExchange (31) : +- * CometColumnarToRow (30) : +- CometHashAggregate (29) @@ -64,7 +64,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -92,7 +92,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -295,10 +295,10 @@ BroadcastExchange (56) +- * CometColumnarToRow (55) +- CometProject (54) +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) + +- CometNativeScan parquet spark_catalog.default.date_dim (52) -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(52) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/extended.txt index ddee139acf..37eea8ba60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/extended.txt @@ -20,15 +20,15 @@ HashAggregate : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- BroadcastExchange : +- CometColumnarToRow : +- CometHashAggregate @@ -44,11 +44,11 @@ HashAggregate : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- BroadcastExchange +- CometColumnarToRow +- CometHashAggregate @@ -64,10 +64,10 @@ HashAggregate : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 55 out of 66 eligible operators (83%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt index f687139735..776f189141 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt @@ -26,15 +26,15 @@ WholeStageCodegen (4) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (1) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt index 2e49d20af4..25b4c305f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt @@ -18,19 +18,19 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) : : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) @@ -44,12 +44,12 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) : : : : : +- * CometColumnarToRow (65) @@ -64,12 +64,12 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) : : : : +- * CometColumnarToRow (86) @@ -84,12 +84,12 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) : : : +- * CometColumnarToRow (107) @@ -104,12 +104,12 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) : : +- * CometColumnarToRow (128) @@ -124,12 +124,12 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) +: : : +- CometNativeScan parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) : +- * CometColumnarToRow (149) @@ -144,12 +144,12 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) +: : +- CometNativeScan parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) +- * CometColumnarToRow (170) @@ -164,16 +164,16 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) + : +- CometNativeScan parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -188,7 +188,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -216,7 +216,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -244,7 +244,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -288,7 +288,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -315,7 +315,7 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -379,7 +379,7 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(47) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -406,7 +406,7 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(53) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -470,7 +470,7 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(68) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -497,7 +497,7 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(74) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -561,7 +561,7 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(89) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -588,7 +588,7 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(95) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -652,7 +652,7 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(110) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -679,7 +679,7 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(116) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -743,7 +743,7 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(131) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -770,7 +770,7 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(137) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -834,7 +834,7 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(152) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -861,7 +861,7 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(158) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/extended.txt index 1e1247665c..24f596de9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/extended.txt @@ -17,19 +17,19 @@ BroadcastNestedLoopJoin : : : : : : : : : +- CometBroadcastHashJoin : : : : : : : : : :- CometProject : : : : : : : : : : +- CometFilter -: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometProject : : : : : : : : : +- CometFilter -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometProject : : : : : : : : +- CometFilter -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : : +- BroadcastExchange : : : : : : +- CometColumnarToRow : : : : : : +- CometHashAggregate @@ -43,19 +43,19 @@ BroadcastNestedLoopJoin : : : : : : : : +- CometBroadcastHashJoin : : : : : : : : :- CometProject : : : : : : : : : +- CometFilter -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometProject : : : : : : : : +- CometFilter -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : +- BroadcastExchange : : : : : +- CometColumnarToRow : : : : : +- CometHashAggregate @@ -69,19 +69,19 @@ BroadcastNestedLoopJoin : : : : : : : +- CometBroadcastHashJoin : : : : : : : :- CometProject : : : : : : : : +- CometFilter -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometHashAggregate @@ -95,19 +95,19 @@ BroadcastNestedLoopJoin : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometHashAggregate @@ -121,19 +121,19 @@ BroadcastNestedLoopJoin : : : : : +- CometBroadcastHashJoin : : : : : :- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : +- CometNativeScan parquet spark_catalog.default.store : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometHashAggregate @@ -147,19 +147,19 @@ BroadcastNestedLoopJoin : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : +- CometNativeScan parquet spark_catalog.default.time_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : +- CometNativeScan parquet spark_catalog.default.store : +- BroadcastExchange : +- CometColumnarToRow : +- CometHashAggregate @@ -173,19 +173,19 @@ BroadcastNestedLoopJoin : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : +- CometNativeScan parquet spark_catalog.default.time_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter -: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: +- CometNativeScan parquet spark_catalog.default.store +- BroadcastExchange +- CometColumnarToRow +- CometHashAggregate @@ -199,18 +199,18 @@ BroadcastNestedLoopJoin : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : +- CometNativeScan parquet spark_catalog.default.time_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 192 out of 206 eligible operators (93%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt index e3923fb3ea..cef8e0e760 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt @@ -19,19 +19,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -48,12 +48,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 @@ -71,12 +71,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 @@ -94,12 +94,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 @@ -117,12 +117,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 @@ -140,12 +140,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 @@ -163,12 +163,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt index 47ce0d4a91..58f1696153 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometNativeScan parquet spark_catalog.default.store (15) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt index 825b1ed81c..77b01e59ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -25,13 +25,13 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 27 out of 33 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt index 61bfd1d960..96c0c76bee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] CometFilter [i_item_sk,i_brand,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] @@ -31,11 +31,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt index b3f32555de..da194f2825 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt @@ -2,10 +2,10 @@ * Project (4) +- * CometColumnarToRow (3) +- CometFilter (2) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) + +- CometNativeScan parquet spark_catalog.default.reason (1) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +(1) CometNativeScan parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -33,10 +33,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + +- CometNativeScan parquet spark_catalog.default.store_sales (5) -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -84,10 +84,10 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + +- CometNativeScan parquet spark_catalog.default.store_sales (13) -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -135,10 +135,10 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) + +- CometNativeScan parquet spark_catalog.default.store_sales (21) -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(21) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -186,10 +186,10 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (29) + +- CometNativeScan parquet spark_catalog.default.store_sales (29) -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(29) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -237,10 +237,10 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (37) + +- CometNativeScan parquet spark_catalog.default.store_sales (37) -(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(37) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/extended.txt index ed71033b26..53ba3252cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/extended.txt @@ -7,7 +7,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -18,7 +18,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -29,7 +29,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -40,7 +40,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -51,11 +51,11 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : +- ReusedSubquery +- CometColumnarToRow +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason + +- CometNativeScan parquet spark_catalog.default.reason Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt index 558f5f4b36..9593d6ebd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 @@ -23,7 +23,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 @@ -49,7 +49,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 @@ -62,10 +62,10 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 CometColumnarToRow InputAdapter CometFilter [r_reason_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt index 08ae744b5a..73b54f439b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt @@ -13,19 +13,19 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) + : +- CometNativeScan parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometHashAggregate (43) @@ -39,16 +39,16 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) + : +- CometNativeScan parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -63,7 +63,7 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -91,7 +91,7 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -119,7 +119,7 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +(16) CometNativeScan parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -163,7 +163,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(26) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -190,7 +190,7 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/extended.txt index aa6c577ed7..977fcbbc6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/extended.txt @@ -12,19 +12,19 @@ Project : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : : +- CometNativeScan parquet spark_catalog.default.time_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page + : +- CometNativeScan parquet spark_catalog.default.web_page +- BroadcastExchange +- CometColumnarToRow +- CometHashAggregate @@ -38,18 +38,18 @@ Project : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : +- CometNativeScan parquet spark_catalog.default.time_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page + +- CometNativeScan parquet spark_catalog.default.web_page Comet accelerated 48 out of 51 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt index 0991e4e0dc..a13072cd91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt index cf532d2b41..30ecac0944 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt @@ -19,32 +19,32 @@ : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometProject (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometFilter (5) : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : : : +- CometNativeScan parquet spark_catalog.default.customer (15) : : +- CometBroadcastExchange (23) : : +- CometProject (22) : : +- CometFilter (21) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) + +- CometNativeScan parquet spark_catalog.default.household_demographics (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(1) CometNativeScan parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -84,7 +84,7 @@ Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -112,7 +112,7 @@ Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(15) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -136,7 +136,7 @@ Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -164,7 +164,7 @@ Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(26) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -192,7 +192,7 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(32) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#25, hd_buy_potential#26] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -252,10 +252,10 @@ BroadcastExchange (48) +- * CometColumnarToRow (47) +- CometProject (46) +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) + +- CometNativeScan parquet spark_catalog.default.date_dim (44) -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(44) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/extended.txt index a1c6b73d37..d08a306fbf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/extended.txt @@ -18,7 +18,7 @@ CometColumnarToRow : : : : : +- CometBroadcastHashJoin : : : : : :- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns @@ -27,25 +27,25 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : +- CometNativeScan parquet spark_catalog.default.customer_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + +- CometNativeScan parquet spark_catalog.default.household_demographics Comet accelerated 45 out of 47 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt index 3e9b8945da..0daeeb561b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt @@ -20,7 +20,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] @@ -31,23 +31,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #7 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk] #9 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt index c317a12697..a9681ae9c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt @@ -14,7 +14,7 @@ : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometNativeScan parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -27,7 +27,7 @@ : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Condition : isnotnull(ws_item_sk#7) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + +- CometNativeScan parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/extended.txt index 20df9a8047..c3eeb08246 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/extended.txt @@ -15,11 +15,11 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -33,10 +33,10 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 35 out of 38 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt index 4ec5755df7..c411db0635 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] @@ -37,5 +37,5 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt index 799d860c95..3f38adcd39 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt @@ -11,19 +11,19 @@ : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : +- CometNativeScan parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) + +- CometNativeScan parquet spark_catalog.default.reason (12) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -73,7 +73,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +(12) CometNativeScan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/extended.txt index 335f2765d7..7203a678b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/extended.txt @@ -10,15 +10,15 @@ CometColumnarToRow : :- CometSort : : +- CometExchange : : +- CometProject - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- CometSort : +- CometExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : +- CometNativeScan parquet spark_catalog.default.store_returns +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason + +- CometNativeScan parquet spark_catalog.default.reason Comet accelerated 21 out of 21 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt index d1de4f3475..c0b74010df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt @@ -12,13 +12,13 @@ WholeStageCodegen (1) CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometExchange [ss_item_sk,ss_ticket_number] #2 CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometExchange [sr_item_sk,sr_ticket_number] #3 CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [r_reason_sk] #4 CometProject [r_reason_sk] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt index 8164e345a6..1caeb8ca76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) + +- CometNativeScan parquet spark_catalog.default.web_site (29) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -92,7 +92,7 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(12) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -115,7 +115,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +(29) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/extended.txt index eac4939621..e5c7b8688a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/extended.txt @@ -18,26 +18,26 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometSort : : : +- CometExchange : : : +- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site + +- CometNativeScan parquet spark_catalog.default.web_site Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt index 629178d106..feab73bcd6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [wr_order_number] CometExchange [wr_order_number] #4 CometProject [wr_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt index 5c319928ea..e49aacd1b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt @@ -18,14 +18,14 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : : +- CometProject (14) : : : : +- CometSortMergeJoin (13) : : : : :- CometSort (10) : : : : : +- CometExchange (9) : : : : : +- CometProject (8) : : : : : +- CometFilter (7) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) : : : : +- CometSort (12) : : : : +- ReusedExchange (11) : : : +- CometProject (28) @@ -34,7 +34,7 @@ : : : : +- CometExchange (19) : : : : +- CometProject (18) : : : : +- CometFilter (17) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (16) : : : +- CometProject (26) : : : +- CometSortMergeJoin (25) : : : :- CometSort (22) @@ -44,18 +44,18 @@ : : +- CometBroadcastExchange (33) : : +- CometProject (32) : : +- CometFilter (31) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) : +- CometBroadcastExchange (39) : +- CometProject (38) : +- CometFilter (37) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) + : +- CometNativeScan parquet spark_catalog.default.customer_address (36) +- CometBroadcastExchange (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) + +- CometNativeScan parquet spark_catalog.default.web_site (42) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -78,7 +78,7 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -122,7 +122,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(16) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -182,7 +182,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#15, d_date#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -210,7 +210,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#15], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#15] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(36) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -238,7 +238,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#17], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#17] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +(42) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#19, web_company_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/extended.txt index 6ff8eba58f..cbba03d28d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/extended.txt @@ -17,49 +17,49 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : : +- CometProject : : : : +- CometSortMergeJoin : : : : :- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometProject : : : +- CometSortMergeJoin : : : :- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : : +- CometProject : : : +- CometSortMergeJoin : : : :- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometSort : : : +- CometExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site + +- CometNativeScan parquet spark_catalog.default.web_site Comet accelerated 59 out of 61 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt index 6c1b1bccfa..15f29c507c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt @@ -21,14 +21,14 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 CometProject [wr_order_number] @@ -37,7 +37,7 @@ WholeStageCodegen (2) CometExchange [wr_order_number] #4 CometProject [wr_order_number] CometFilter [wr_order_number,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] @@ -47,12 +47,12 @@ WholeStageCodegen (2) CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt index f90d8b12b9..db701aae34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt @@ -11,22 +11,22 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + : +- CometNativeScan parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) + +- CometNativeScan parquet spark_catalog.default.store (16) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -69,7 +69,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -97,7 +97,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/extended.txt index 0f623c9021..170d3b7879 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/extended.txt @@ -10,18 +10,18 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : +- CometNativeScan parquet spark_catalog.default.time_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt index bbbd07245e..b40f41659f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt index 420f37fccc..5e84af4892 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt @@ -15,7 +15,7 @@ : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) + : +- CometNativeScan parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) @@ -33,7 +33,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] ReadSchema: struct -(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(2) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -149,10 +149,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + +- CometNativeScan parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/extended.txt index e1aefb8382..355bb7fda2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/extended.txt @@ -16,11 +16,11 @@ CometColumnarToRow : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometSort +- CometHashAggregate +- CometExchange @@ -32,6 +32,6 @@ CometColumnarToRow +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 30 out of 33 eligible operators (90%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt index a7de047b25..3088193cfb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt @@ -20,11 +20,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] CometExchange [cs_bill_customer_sk,cs_item_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt index 4cc725ef2b..1eacde21e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt @@ -20,11 +20,11 @@ : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -67,7 +67,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -150,10 +150,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) + +- CometNativeScan parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(27) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt index 4c972848e7..07eca8ca5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt @@ -21,14 +21,14 @@ CometColumnarToRow : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 25 out of 29 eligible operators (86%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt index af05a7d8c6..94361b16ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt @@ -29,12 +29,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt index 7e57135c10..3447b6efa3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt @@ -13,24 +13,24 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) : +- CometBroadcastExchange (16) : +- CometFilter (15) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- CometNativeScan parquet spark_catalog.default.call_center (14) +- CometBroadcastExchange (22) +- CometProject (21) +- CometFilter (20) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -41,7 +41,7 @@ ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customer_sk#13], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(24) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#19, ca_county#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#19], Inner, BuildRight Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#19] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(30) CometNativeScan parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_education_status#24, cd_purchase_estimate#25, cd_credit_rating#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/extended.txt index 2cdc75e15e..90b468da99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/extended.txt @@ -11,7 +11,7 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -21,11 +21,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometUnion : : :- CometProject @@ -35,7 +35,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -43,14 +43,14 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt index 38f41d10a5..031c06de9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -25,11 +25,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [customer_sk] #5 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] @@ -45,8 +45,8 @@ WholeStageCodegen (1) CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt index 49c1c8e0d7..47b7928d4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,7 +51,7 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#23, year_total#24] Right output [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -325,7 +325,7 @@ Arguments: [customer_id#23], [customer_id#68], Inner, BuildRight Input [9]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, customer_id#68, year_total#69] Arguments: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69], [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/extended.txt index 476c7be954..18c1c90d20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -23,10 +23,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -37,7 +37,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -45,10 +45,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -60,14 +60,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -78,13 +78,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 79 out of 85 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt index 157d1d587e..26364c9ff3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -27,10 +27,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -51,10 +51,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -66,7 +66,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt index 410aecff1b..e1a264a366 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject (23) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt index 3f41c97ff5..282edf0ce0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt index 58ac81ba0d..36cef3bca1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt index 33224acffb..e2609155db 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt @@ -17,7 +17,7 @@ : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -32,7 +32,7 @@ : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) @@ -42,11 +42,11 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -60,12 +60,12 @@ : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -83,7 +83,7 @@ +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -121,7 +121,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#60], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(72) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(103) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(108) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) + +- CometNativeScan parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(112) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#99] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) + +- CometNativeScan parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(117) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) + +- CometNativeScan parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(122) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/extended.txt index f56d229b68..5011719f37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -23,7 +23,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -31,7 +31,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -51,13 +51,13 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -74,11 +74,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -89,15 +89,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -108,20 +108,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -138,11 +138,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -153,15 +153,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -172,16 +172,16 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter : +- ReusedSubquery @@ -204,13 +204,13 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -227,11 +227,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -242,15 +242,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -261,20 +261,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -291,11 +291,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -306,15 +306,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -325,15 +325,15 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter : +- ReusedSubquery - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 298 out of 327 eligible operators (91%). Final plan contains 10 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt index fb9abae378..4ae30a0feb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -75,11 +75,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -90,11 +90,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -109,13 +109,13 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt index 4ac6c99894..e1d8dbbda8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt @@ -24,7 +24,7 @@ : : : : +- CometProject (45) : : : : +- CometBroadcastHashJoin (44) : : : : :- CometFilter (4) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) @@ -39,7 +39,7 @@ : : : : : : +- CometBroadcastExchange (24) : : : : : : +- CometBroadcastHashJoin (23) : : : : : : :- CometFilter (8) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : : +- CometBroadcastExchange (22) : : : : : : +- CometProject (21) : : : : : : +- CometBroadcastHashJoin (20) @@ -49,11 +49,11 @@ : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : : +- CometBroadcastExchange (13) : : : : : : : +- CometFilter (12) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : : +- CometBroadcastExchange (19) : : : : : : +- CometProject (18) : : : : : : +- CometFilter (17) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : : +- ReusedExchange (27) : : : : +- CometBroadcastExchange (41) : : : : +- CometProject (40) @@ -67,12 +67,12 @@ : : : +- CometBroadcastExchange (52) : : : +- CometBroadcastHashJoin (51) : : : :- CometFilter (49) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : : +- ReusedExchange (50) : : +- CometBroadcastExchange (58) : : +- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (55) : :- CometFilter (78) : : +- CometHashAggregate (77) : : +- CometExchange (76) @@ -135,7 +135,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -158,7 +158,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -181,7 +181,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -205,7 +205,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -348,7 +348,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -380,7 +380,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -703,7 +703,7 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer : +- CometBroadcastExchange (131) : +- CometProject (130) : +- CometFilter (129) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (128) + : +- CometNativeScan parquet spark_catalog.default.date_dim (128) +- CometProject (137) +- CometBroadcastHashJoin (136) :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (134) @@ -736,7 +736,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#130), dynamicpruningexpression(cs_sold_date_sk#130 IN dynamicpruning#131)] ReadSchema: struct -(128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(128) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -812,10 +812,10 @@ BroadcastExchange (147) +- * CometColumnarToRow (146) +- CometProject (145) +- CometFilter (144) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) + +- CometNativeScan parquet spark_catalog.default.date_dim (143) -(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(143) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -844,10 +844,10 @@ BroadcastExchange (152) +- * CometColumnarToRow (151) +- CometProject (150) +- CometFilter (149) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (148) + +- CometNativeScan parquet spark_catalog.default.date_dim (148) -(148) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(148) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -874,10 +874,10 @@ BroadcastExchange (157) +- * CometColumnarToRow (156) +- CometProject (155) +- CometFilter (154) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (153) + +- CometNativeScan parquet spark_catalog.default.date_dim (153) -(153) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(153) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#146] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/extended.txt index 83deb45432..6c5a02babc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/extended.txt @@ -22,7 +22,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -31,11 +31,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -43,7 +43,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -59,12 +59,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -81,11 +81,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -96,15 +96,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -115,20 +115,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -145,11 +145,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -160,15 +160,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -179,15 +179,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -205,7 +205,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -222,11 +222,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -237,15 +237,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -256,20 +256,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -286,11 +286,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -301,15 +301,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -320,15 +320,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -346,7 +346,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -363,11 +363,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -378,15 +378,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -397,20 +397,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -427,11 +427,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -442,15 +442,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -461,15 +461,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -491,7 +491,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -500,11 +500,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -512,7 +512,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -528,12 +528,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -550,11 +550,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -565,15 +565,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -584,20 +584,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -614,11 +614,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -629,15 +629,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -648,15 +648,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -674,7 +674,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -691,11 +691,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -706,15 +706,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -725,20 +725,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -755,11 +755,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -770,15 +770,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -789,15 +789,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -815,7 +815,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -832,11 +832,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -847,15 +847,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -866,20 +866,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -896,11 +896,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -911,15 +911,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -930,15 +930,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -960,7 +960,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -969,11 +969,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -981,7 +981,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -997,12 +997,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1019,11 +1019,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1034,15 +1034,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1053,20 +1053,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1083,11 +1083,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1098,15 +1098,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1117,15 +1117,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -1143,7 +1143,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1160,11 +1160,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1175,15 +1175,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1194,20 +1194,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1224,11 +1224,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1239,15 +1239,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1258,15 +1258,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -1284,7 +1284,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1301,11 +1301,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1316,15 +1316,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1335,20 +1335,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -1365,11 +1365,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1380,15 +1380,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -1399,15 +1399,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -1429,7 +1429,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -1438,11 +1438,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -1450,7 +1450,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -1466,12 +1466,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1488,11 +1488,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1503,15 +1503,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1522,20 +1522,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1552,11 +1552,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1567,15 +1567,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1586,15 +1586,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -1612,7 +1612,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1629,11 +1629,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1644,15 +1644,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1663,20 +1663,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1693,11 +1693,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1708,15 +1708,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1727,15 +1727,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -1753,7 +1753,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1770,11 +1770,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1785,15 +1785,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1804,20 +1804,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -1834,11 +1834,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1849,15 +1849,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -1868,15 +1868,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -1898,7 +1898,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -1907,11 +1907,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -1919,7 +1919,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -1935,12 +1935,12 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1957,11 +1957,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1972,15 +1972,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1991,20 +1991,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -2021,11 +2021,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -2036,15 +2036,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -2055,15 +2055,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -2081,7 +2081,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -2098,11 +2098,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -2113,15 +2113,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -2132,20 +2132,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -2162,11 +2162,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -2177,15 +2177,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -2196,15 +2196,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometFilter : +- ReusedSubquery +- CometHashAggregate @@ -2222,7 +2222,7 @@ CometColumnarToRow : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -2239,11 +2239,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -2254,15 +2254,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -2273,20 +2273,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -2303,11 +2303,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -2318,15 +2318,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -2337,14 +2337,14 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 2127 out of 2302 eligible operators (92%). Final plan contains 46 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt index 758e917335..8f7b98b200 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt @@ -34,11 +34,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] @@ -61,12 +61,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #5 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #6 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -85,11 +85,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -100,11 +100,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 CometProject [i_brand_id,i_class_id,i_category_id] @@ -119,12 +119,12 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #5 CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt index 86072fd830..87935c8001 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt @@ -22,26 +22,26 @@ : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + : +- CometNativeScan parquet spark_catalog.default.item (32) :- CometHashAggregate (66) : +- CometExchange (65) : +- CometHashAggregate (64) @@ -65,7 +65,7 @@ : : : +- CometBroadcastExchange (55) : : : +- CometProject (54) : : : +- CometFilter (53) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (52) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (52) : : +- ReusedExchange (58) : +- ReusedExchange (61) :- CometHashAggregate (92) @@ -91,7 +91,7 @@ : : : +- CometBroadcastExchange (81) : : : +- CometProject (80) : : : +- CometFilter (79) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (78) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (78) : : +- ReusedExchange (84) : +- ReusedExchange (87) :- CometHashAggregate (118) @@ -117,7 +117,7 @@ : : : +- CometBroadcastExchange (107) : : : +- CometProject (106) : : : +- CometFilter (105) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (104) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (104) : : +- ReusedExchange (110) : +- ReusedExchange (113) +- CometHashAggregate (143) @@ -144,7 +144,7 @@ : +- ReusedExchange (133) +- CometBroadcastExchange (138) +- CometFilter (137) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (136) + +- CometNativeScan parquet spark_catalog.default.item (136) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -159,7 +159,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt index cd52b2cd12..754a4fad60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt index cf18e68a3d..6fadc67351 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt index f85c898208..161f471629 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt @@ -17,14 +17,14 @@ TakeOrderedAndProject (26) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometNativeScan parquet spark_catalog.default.item (9) +- BroadcastExchange (18) +- * CometColumnarToRow (17) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) + +- CometNativeScan parquet spark_catalog.default.warehouse (16) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -67,7 +67,7 @@ Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_pro (15) CometColumnarToRow [codegen id : 2] Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(16) CometNativeScan parquet spark_catalog.default.warehouse Output: [] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) + +- CometNativeScan parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(27) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/extended.txt index 6cb89465d2..dfdad622b8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/extended.txt @@ -18,17 +18,17 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- BroadcastExchange +- CometColumnarToRow - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + +- CometNativeScan parquet spark_catalog.default.warehouse Comet accelerated 19 out of 28 eligible operators (67%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt index 221c6063ce..16246ec037 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt @@ -24,18 +24,18 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + CometNativeScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt index 189a54b584..2b6d1f0a22 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt @@ -18,14 +18,14 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : : +- CometNativeScan parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) + : +- CometNativeScan parquet spark_catalog.default.warehouse (15) :- CometHashAggregate (29) : +- CometExchange (28) : +- CometHashAggregate (27) @@ -60,7 +60,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/extended.txt index 3d9d23d2cc..20468961d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/extended.txt @@ -20,22 +20,22 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : +- CometNativeScan parquet spark_catalog.default.household_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 35 out of 37 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt index d57afd4fcf..a12adcdfec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt index 5df7e6870f..d44320a8a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt @@ -15,7 +15,7 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) @@ -23,7 +23,7 @@ TakeOrderedAndProject (47) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) @@ -40,15 +40,15 @@ TakeOrderedAndProject (47) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -66,7 +66,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(29) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#20] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18, ca_state#20] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(36) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) + +- CometNativeScan parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(48) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/extended.txt index a6f33d6f7e..ee476d727a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/extended.txt @@ -14,7 +14,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -24,11 +24,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -38,7 +38,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -48,16 +48,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt index fea6a32741..24572a88a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt index 97c7d7659b..9bf35ea541 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt @@ -12,7 +12,7 @@ : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) @@ -20,7 +20,7 @@ : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) @@ -34,14 +34,14 @@ : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) +- CometBroadcastExchange (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,7 +59,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customsk#13], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(24) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#19, ca_state#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#19], Inner, BuildRight Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#19, ca_state#21] Arguments: [c_current_cdemo_sk#2, ca_state#21], [c_current_cdemo_sk#2, ca_state#21] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(30) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/extended.txt index 2cdc75e15e..90b468da99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/extended.txt @@ -11,7 +11,7 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -21,11 +21,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometUnion : : :- CometProject @@ -35,7 +35,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -43,14 +43,14 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt index 69e8d4868c..33cc09317c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -25,11 +25,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [customsk] #5 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] @@ -45,8 +45,8 @@ WholeStageCodegen (1) CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt index 0f966424f8..406ec3805f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt @@ -23,15 +23,15 @@ TakeOrderedAndProject (43) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : : +- CometNativeScan parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- CometNativeScan parquet spark_catalog.default.store (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -56,7 +56,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -418,10 +418,10 @@ BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometProject (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt index f7d3371108..c098b443fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt @@ -36,11 +36,11 @@ TakeOrderedAndProject : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -59,11 +59,11 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometSort : +- CometColumnarExchange : +- HashAggregate @@ -88,7 +88,7 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -106,7 +106,7 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -141,11 +141,11 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -164,11 +164,11 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometSort +- CometColumnarExchange +- HashAggregate @@ -193,7 +193,7 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -211,6 +211,6 @@ TakeOrderedAndProject +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 138 out of 196 eligible operators (70%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt index b3013059b0..7e155be3c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt @@ -48,11 +48,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter BroadcastExchange #8 WholeStageCodegen (3) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt index d1daee98c1..7285a59f3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(14) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt index de7d26cd06..3ab8670a2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -30,13 +30,13 @@ TakeOrderedAndProject : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : : +- CometNativeScan parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -54,7 +54,7 @@ TakeOrderedAndProject : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -62,13 +62,13 @@ TakeOrderedAndProject : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : +- CometNativeScan parquet spark_catalog.default.call_center +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -86,7 +86,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -94,12 +94,12 @@ TakeOrderedAndProject : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + +- CometNativeScan parquet spark_catalog.default.call_center Comet accelerated 75 out of 97 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt index d655789fe6..1c40ce03de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] @@ -38,13 +38,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt index 306485ca63..fd038ed439 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt @@ -26,11 +26,11 @@ : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : : +- CometNativeScan parquet spark_catalog.default.store (14) : :- CometHashAggregate (41) : : +- CometExchange (40) : : +- CometHashAggregate (39) @@ -49,7 +49,7 @@ : : +- CometBroadcastExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (33) + : : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) : +- CometHashAggregate (64) : +- CometExchange (63) : +- CometHashAggregate (62) @@ -67,12 +67,12 @@ : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (45) : : : +- CometProject (49) : : : +- CometFilter (48) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) : : +- ReusedExchange (53) : +- CometBroadcastExchange (59) : +- CometProject (58) : +- CometFilter (57) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (56) + : +- CometNativeScan parquet spark_catalog.default.web_site (56) :- CometHashAggregate (73) : +- CometExchange (72) : +- CometHashAggregate (71) @@ -121,7 +121,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -149,7 +149,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -239,7 +239,7 @@ Arguments: [date_sk#37], [d_date_sk#52], Inner, BuildRight Input [7]: [page_sk#36, date_sk#37, sales_price#38, profit#39, return_amt#40, net_loss#41, d_date_sk#52] Arguments: [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41], [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41] -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +(33) CometNativeScan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -308,7 +308,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(22) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) + +- CometNativeScan parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(46) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#11, d_year#24, d_moy#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/extended.txt index f5b69fc6cf..5cce29dd89 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/extended.txt @@ -14,10 +14,10 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -33,18 +33,18 @@ CometColumnarToRow : : : +- CometHashAggregate : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -52,6 +52,6 @@ CometColumnarToRow +- CometHashAggregate +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt index 2978e30c1c..04824dfed4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt @@ -16,10 +16,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt index 077daeca3a..e9a14a21a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt @@ -49,7 +49,7 @@ : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,51 +62,51 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (31) : : : : : : : : : : : : : : +- CometBroadcastExchange (39) : : : : : : : : : : : : : : +- CometProject (38) : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (36) : : : : : : : : : : : : : +- CometBroadcastExchange (44) : : : : : : : : : : : : : +- CometFilter (43) - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (42) : : : : : : : : : : : : +- CometBroadcastExchange (49) : : : : : : : : : : : : +- CometFilter (48) - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (47) : : : : : : : : : : : +- ReusedExchange (52) : : : : : : : : : : +- CometBroadcastExchange (58) : : : : : : : : : : +- CometProject (57) : : : : : : : : : : +- CometFilter (56) - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (55) : : : : : : : : : +- ReusedExchange (61) : : : : : : : : +- CometBroadcastExchange (66) : : : : : : : : +- CometFilter (65) - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (64) : : : : : : : +- CometBroadcastExchange (71) : : : : : : : +- CometFilter (70) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (69) : : : : : : +- ReusedExchange (74) : : : : : +- CometBroadcastExchange (80) : : : : : +- CometProject (79) : : : : : +- CometFilter (78) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (77) : : : : +- ReusedExchange (83) : : : +- CometBroadcastExchange (88) : : : +- CometFilter (87) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (86) : : +- ReusedExchange (91) : +- CometBroadcastExchange (97) : +- CometProject (96) : +- CometFilter (95) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) + : +- CometNativeScan parquet spark_catalog.default.item (94) +- CometSort (171) +- CometExchange (170) +- CometHashAggregate (169) @@ -152,7 +152,7 @@ : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (107) : : : : : : : : : : : : : : : +- CometSort (118) : : : : : : : : : : : : : : : +- CometProject (117) : : : : : : : : : : : : : : : +- CometFilter (116) @@ -160,7 +160,7 @@ : : : : : : : : : : : : : : : +- ReusedExchange (114) : : : : : : : : : : : : : : +- CometBroadcastExchange (123) : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (121) : : : : : : : : : : : : : +- ReusedExchange (126) : : : : : : : : : : : : +- ReusedExchange (129) : : : : : : : : : : : +- ReusedExchange (132) @@ -193,7 +193,7 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -225,7 +225,7 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(11) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -248,7 +248,7 @@ Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -315,7 +315,7 @@ Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -339,7 +339,7 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(36) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -367,7 +367,7 @@ Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(42) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -391,7 +391,7 @@ Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(47) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -427,7 +427,7 @@ Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(55) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#48, cd_marital_status#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -467,7 +467,7 @@ Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_stat Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(64) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#53] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -491,7 +491,7 @@ Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(69) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -527,7 +527,7 @@ Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(77) CometNativeScan parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -567,7 +567,7 @@ Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +(86) CometNativeScan parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#70] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -603,7 +603,7 @@ Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(94) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -665,7 +665,7 @@ Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(107) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -726,7 +726,7 @@ Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(121) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -962,10 +962,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dy BroadcastExchange (180) +- * CometColumnarToRow (179) +- CometFilter (178) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) + +- CometNativeScan parquet spark_catalog.default.date_dim (177) -(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(177) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -987,10 +987,10 @@ Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN BroadcastExchange (184) +- * CometColumnarToRow (183) +- CometFilter (182) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) + +- CometNativeScan parquet spark_catalog.default.date_dim (181) -(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(181) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/extended.txt index 059acab385..af08477984 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/extended.txt @@ -50,10 +50,10 @@ CometColumnarToRow : : : : : : : : : : : : : : : : : : +- BroadcastExchange : : : : : : : : : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter @@ -66,63 +66,63 @@ CometColumnarToRow : : : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : +- CometProject : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometProject : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : : : +- CometNativeScan parquet spark_catalog.default.income_band : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : : +- CometNativeScan parquet spark_catalog.default.income_band : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometSort +- CometExchange +- CometHashAggregate @@ -170,10 +170,10 @@ CometColumnarToRow : : : : : : : : : : : : : : : : : +- BroadcastExchange : : : : : : : : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : +- CometFilter @@ -186,62 +186,62 @@ CometColumnarToRow : : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometProject : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometProject : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : : +- CometNativeScan parquet spark_catalog.default.income_band : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : +- CometNativeScan parquet spark_catalog.default.income_band +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 238 out of 242 eligible operators (98%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt index 46fe063430..87b73a92e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt @@ -54,10 +54,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -70,51 +70,51 @@ WholeStageCodegen (1) CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #12 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #12 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #13 CometBroadcastExchange [p_promo_sk] #14 CometFilter [p_promo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometBroadcastExchange [ib_income_band_sk] #17 CometFilter [ib_income_band_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] ReusedExchange [ib_income_band_sk] #17 CometBroadcastExchange [i_item_sk,i_product_name] #18 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] @@ -164,10 +164,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -175,7 +175,7 @@ WholeStageCodegen (1) ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 CometBroadcastExchange [d_date_sk,d_year] #23 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #10 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 ReusedExchange [d_date_sk,d_year] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt index ac04f86f96..5f872e5a8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt @@ -20,15 +20,15 @@ TakeOrderedAndProject (70) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometNativeScan parquet spark_catalog.default.item (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -109,7 +109,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -135,7 +135,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -373,10 +373,10 @@ BroadcastExchange (70) +- * CometColumnarToRow (69) +- CometProject (68) +- CometFilter (67) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) + +- CometNativeScan parquet spark_catalog.default.date_dim (66) -(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(66) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt index a677fed73a..1adcf0f683 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt @@ -24,17 +24,17 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- BroadcastHashJoin : :- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- BroadcastExchange : +- Project : +- Filter @@ -54,11 +54,11 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- HashAggregate : +- CometColumnarToRow : +- CometColumnarExchange @@ -79,17 +79,17 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- BroadcastHashJoin : :- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- BroadcastExchange : +- Project : +- Filter @@ -109,11 +109,11 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- HashAggregate +- CometColumnarToRow +- CometColumnarExchange @@ -134,17 +134,17 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- BroadcastHashJoin :- CometColumnarToRow : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- BroadcastExchange +- Project +- Filter @@ -164,10 +164,10 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 102 out of 153 eligible operators (66%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt index 8de8b7e637..fe4fb64d3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt @@ -36,11 +36,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) @@ -49,7 +49,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (2) @@ -74,7 +74,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastExchange [s_store_sk,s_state] #9 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #5 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt index ee3242f2b1..41ef5269e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt @@ -30,39 +30,39 @@ : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) : : : : : : : +- CometBroadcastExchange (15) : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) : : : : : : +- CometBroadcastExchange (21) : : : : : : +- CometProject (20) : : : : : : +- CometFilter (19) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) : : : : : +- CometBroadcastExchange (27) : : : : : +- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) : : : : +- CometBroadcastExchange (33) : : : : +- CometProject (32) : : : : +- CometFilter (31) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) : : : +- CometBroadcastExchange (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) : : +- CometBroadcastExchange (43) : : +- CometFilter (42) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) : +- CometBroadcastExchange (48) : +- CometFilter (47) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) +- CometSort (57) +- CometExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -77,10 +77,10 @@ ReadSchema: struct @@ -102,7 +102,7 @@ Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < c Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -126,7 +126,7 @@ Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#16, i_item_desc#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -150,7 +150,7 @@ Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#18, cd_marital_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -178,7 +178,7 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#20, hd_buy_potential#21] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -206,7 +206,7 @@ Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -234,7 +234,7 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -258,7 +258,7 @@ Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#28, d_date#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -282,7 +282,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) + +- CometNativeScan parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(65) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/extended.txt index 9000d27d1b..4f2941b209 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/extended.txt @@ -32,41 +32,41 @@ CometColumnarToRow : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : +- CometProject : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion +- CometSort +- CometExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + +- CometNativeScan parquet spark_catalog.default.catalog_returns Comet accelerated 66 out of 68 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt index 678f8c67df..e7af25252a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt index b0ae8bb695..280f5e12fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,7 +51,7 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/extended.txt index 476c7be954..18c1c90d20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -23,10 +23,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -37,7 +37,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -45,10 +45,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -60,14 +60,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -78,13 +78,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 79 out of 85 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt index 72e39422e0..6411ab4b43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -27,10 +27,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -51,10 +51,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] @@ -66,7 +66,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index 6925eb301e..e7f56d9c92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -26,15 +26,15 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -51,7 +51,7 @@ : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -68,7 +68,7 @@ : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -92,7 +92,7 @@ : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -137,7 +137,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -78,7 +78,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -213,10 +213,10 @@ BroadcastExchange (42) +- * CometColumnarToRow (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) + +- CometNativeScan parquet spark_catalog.default.date_dim (38) -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(38) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt index 46c47555a8..0a00a8c60b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt @@ -22,15 +22,15 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -48,15 +48,15 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -74,14 +74,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 72 out of 81 eligible operators (88%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt index 11e3c03d80..fd3b3cf399 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt @@ -28,15 +28,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #6 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] CometExchange [i_category] #7 CometHashAggregate [total_sum] [i_category,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt index 96a0ec392e..62d0895b96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt @@ -19,11 +19,11 @@ : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + +- CometNativeScan parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt index 3dbaf2e346..e9f812d706 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt @@ -20,14 +20,14 @@ CometColumnarToRow : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt index 46191f59cd..af593e1269 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt @@ -28,12 +28,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] From afc561ee47827eea8fb178ab5a2eef93c3b58106 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Dec 2025 10:00:45 -0700 Subject: [PATCH 6/8] golden files --- .../q1/explain.txt | 16 +- .../q1/extended.txt | 10 +- .../q1/simplified.txt | 8 +- .../q10/explain.txt | 20 +- .../q10/extended.txt | 14 +- .../q10/simplified.txt | 10 +- .../q11/explain.txt | 32 +- .../q11/extended.txt | 20 +- .../q11/simplified.txt | 16 +- .../q12/explain.txt | 12 +- .../q12/extended.txt | 6 +- .../q12/simplified.txt | 6 +- .../q13/explain.txt | 24 +- .../q13/extended.txt | 12 +- .../q13/simplified.txt | 12 +- .../q14a/explain.txt | 32 +- .../q14a/extended.txt | 116 ++-- .../q14a/simplified.txt | 16 +- .../q14b/explain.txt | 48 +- .../q14b/extended.txt | 86 +-- .../q14b/simplified.txt | 24 +- .../q15/explain.txt | 16 +- .../q15/extended.txt | 8 +- .../q15/simplified.txt | 8 +- .../q16/explain.txt | 24 +- .../q16/extended.txt | 12 +- .../q16/simplified.txt | 12 +- .../q17/explain.txt | 24 +- .../q17/extended.txt | 14 +- .../q17/simplified.txt | 12 +- .../q18/explain.txt | 28 +- .../q18/extended.txt | 14 +- .../q18/simplified.txt | 14 +- .../q19/explain.txt | 26 +- .../q19/extended.txt | 12 +- .../q19/simplified.txt | 12 +- .../q2/explain.txt | 24 +- .../q2/extended.txt | 16 +- .../q2/simplified.txt | 10 +- .../q20/explain.txt | 12 +- .../q20/extended.txt | 6 +- .../q20/simplified.txt | 6 +- .../q21/explain.txt | 16 +- .../q21/extended.txt | 8 +- .../q21/simplified.txt | 8 +- .../q22/explain.txt | 16 +- .../q22/extended.txt | 8 +- .../q22/simplified.txt | 8 +- .../q23a/explain.txt | 36 +- .../q23a/extended.txt | 32 +- .../q23a/simplified.txt | 18 +- .../q23b/explain.txt | 40 +- .../q23b/extended.txt | 44 +- .../q23b/simplified.txt | 20 +- .../q24a/explain.txt | 28 +- .../q24a/extended.txt | 24 +- .../q24a/simplified.txt | 14 +- .../q24b/explain.txt | 28 +- .../q24b/extended.txt | 24 +- .../q24b/simplified.txt | 14 +- .../q25/explain.txt | 24 +- .../q25/extended.txt | 14 +- .../q25/simplified.txt | 12 +- .../q26/explain.txt | 20 +- .../q26/extended.txt | 10 +- .../q26/simplified.txt | 10 +- .../q27/explain.txt | 20 +- .../q27/extended.txt | 10 +- .../q27/simplified.txt | 10 +- .../q28/explain.txt | 24 +- .../q28/extended.txt | 12 +- .../q28/simplified.txt | 12 +- .../q29/explain.txt | 32 +- .../q29/extended.txt | 16 +- .../q29/simplified.txt | 16 +- .../q3/explain.txt | 14 +- .../q3/extended.txt | 6 +- .../q3/simplified.txt | 6 +- .../q30/explain.txt | 20 +- .../q30/extended.txt | 14 +- .../q30/simplified.txt | 10 +- .../q31/explain.txt | 28 +- .../q31/extended.txt | 30 +- .../q31/simplified.txt | 14 +- .../q32/explain.txt | 12 +- .../q32/extended.txt | 8 +- .../q32/simplified.txt | 6 +- .../q33/explain.txt | 20 +- .../q33/extended.txt | 26 +- .../q33/simplified.txt | 10 +- .../q34/explain.txt | 20 +- .../q34/extended.txt | 10 +- .../q34/simplified.txt | 10 +- .../q35/explain.txt | 20 +- .../q35/extended.txt | 14 +- .../q35/simplified.txt | 10 +- .../q36/explain.txt | 16 +- .../q36/extended.txt | 8 +- .../q36/simplified.txt | 8 +- .../q37/explain.txt | 16 +- .../q37/extended.txt | 8 +- .../q37/simplified.txt | 8 +- .../q38/explain.txt | 12 +- .../q38/extended.txt | 14 +- .../q38/simplified.txt | 6 +- .../q39a/explain.txt | 24 +- .../q39a/extended.txt | 16 +- .../q39a/simplified.txt | 12 +- .../q39b/explain.txt | 24 +- .../q39b/extended.txt | 16 +- .../q39b/simplified.txt | 12 +- .../q4/explain.txt | 40 +- .../q4/extended.txt | 28 +- .../q4/simplified.txt | 20 +- .../q40/explain.txt | 20 +- .../q40/extended.txt | 10 +- .../q40/simplified.txt | 10 +- .../q41/explain.txt | 8 +- .../q41/extended.txt | 4 +- .../q41/simplified.txt | 4 +- .../q42/explain.txt | 14 +- .../q42/extended.txt | 6 +- .../q42/simplified.txt | 6 +- .../q43/explain.txt | 14 +- .../q43/extended.txt | 6 +- .../q43/simplified.txt | 6 +- .../q44/explain.txt | 12 +- .../q44/extended.txt | 10 +- .../q44/simplified.txt | 6 +- .../q45/explain.txt | 24 +- .../q45/extended.txt | 12 +- .../q45/simplified.txt | 12 +- .../q46/explain.txt | 24 +- .../q46/extended.txt | 14 +- .../q46/simplified.txt | 12 +- .../q47/explain.txt | 16 +- .../q47/extended.txt | 24 +- .../q47/simplified.txt | 8 +- .../q48/explain.txt | 20 +- .../q48/extended.txt | 10 +- .../q48/simplified.txt | 10 +- .../q49/explain.txt | 20 +- .../q49/extended.txt | 14 +- .../q49/simplified.txt | 10 +- .../q5/explain.txt | 24 +- .../q5/extended.txt | 16 +- .../q5/simplified.txt | 12 +- .../q50/explain.txt | 22 +- .../q50/extended.txt | 10 +- .../q50/simplified.txt | 10 +- .../q51/explain.txt | 8 +- .../q51/extended.txt | 6 +- .../q51/simplified.txt | 4 +- .../q52/explain.txt | 14 +- .../q52/extended.txt | 6 +- .../q52/simplified.txt | 6 +- .../q53/explain.txt | 16 +- .../q53/extended.txt | 8 +- .../q53/simplified.txt | 8 +- .../q54/explain.txt | 40 +- .../q54/extended.txt | 20 +- .../q54/simplified.txt | 20 +- .../q55/explain.txt | 14 +- .../q55/extended.txt | 6 +- .../q55/simplified.txt | 6 +- .../q56/explain.txt | 20 +- .../q56/extended.txt | 26 +- .../q56/simplified.txt | 10 +- .../q57/explain.txt | 16 +- .../q57/extended.txt | 24 +- .../q57/simplified.txt | 8 +- .../q58/explain.txt | 20 +- .../q58/extended.txt | 30 +- .../q58/simplified.txt | 10 +- .../q59/explain.txt | 26 +- .../q59/extended.txt | 16 +- .../q59/simplified.txt | 12 +- .../q6/explain.txt | 28 +- .../q6/extended.txt | 14 +- .../q6/simplified.txt | 14 +- .../q60/explain.txt | 20 +- .../q60/extended.txt | 26 +- .../q60/simplified.txt | 10 +- .../q61/explain.txt | 28 +- .../q61/extended.txt | 24 +- .../q61/simplified.txt | 14 +- .../q62/explain.txt | 20 +- .../q62/extended.txt | 10 +- .../q62/simplified.txt | 10 +- .../q63/explain.txt | 16 +- .../q63/extended.txt | 8 +- .../q63/simplified.txt | 8 +- .../q64/explain.txt | 68 +- .../q64/extended.txt | 76 +-- .../q64/simplified.txt | 34 +- .../q65/explain.txt | 16 +- .../q65/extended.txt | 10 +- .../q65/simplified.txt | 8 +- .../q66/explain.txt | 20 +- .../q66/extended.txt | 18 +- .../q66/simplified.txt | 10 +- .../q67/explain.txt | 16 +- .../q67/extended.txt | 8 +- .../q67/simplified.txt | 8 +- .../q68/explain.txt | 24 +- .../q68/extended.txt | 14 +- .../q68/simplified.txt | 12 +- .../q69/explain.txt | 20 +- .../q69/extended.txt | 14 +- .../q69/simplified.txt | 10 +- .../q7/explain.txt | 20 +- .../q7/extended.txt | 10 +- .../q7/simplified.txt | 10 +- .../q70/explain.txt | 16 +- .../q70/extended.txt | 10 +- .../q70/simplified.txt | 8 +- .../q71/explain.txt | 16 +- .../q71/extended.txt | 12 +- .../q71/simplified.txt | 8 +- .../q72/explain.txt | 46 +- .../q72/extended.txt | 22 +- .../q72/simplified.txt | 22 +- .../q73/explain.txt | 20 +- .../q73/extended.txt | 10 +- .../q73/simplified.txt | 10 +- .../q74/explain.txt | 32 +- .../q74/extended.txt | 20 +- .../q74/simplified.txt | 16 +- .../q75/explain.txt | 32 +- .../q75/extended.txt | 40 +- .../q75/simplified.txt | 16 +- .../q76/explain.txt | 26 +- .../q76/extended.txt | 18 +- .../q76/simplified.txt | 10 +- .../q77/explain.txt | 16 +- .../q77/extended.txt | 22 +- .../q77/simplified.txt | 8 +- .../q78/explain.txt | 20 +- .../q78/extended.txt | 14 +- .../q78/simplified.txt | 10 +- .../q79/explain.txt | 20 +- .../q79/extended.txt | 10 +- .../q79/simplified.txt | 10 +- .../q8/explain.txt | 24 +- .../q8/extended.txt | 12 +- .../q8/simplified.txt | 12 +- .../q80/explain.txt | 40 +- .../q80/extended.txt | 32 +- .../q80/simplified.txt | 20 +- .../q81/explain.txt | 20 +- .../q81/extended.txt | 14 +- .../q81/simplified.txt | 10 +- .../q82/explain.txt | 16 +- .../q82/extended.txt | 8 +- .../q82/simplified.txt | 8 +- .../q83/explain.txt | 20 +- .../q83/extended.txt | 30 +- .../q83/simplified.txt | 10 +- .../q84/explain.txt | 24 +- .../q84/extended.txt | 12 +- .../q84/simplified.txt | 12 +- .../q85/explain.txt | 32 +- .../q85/extended.txt | 16 +- .../q85/simplified.txt | 16 +- .../q86/explain.txt | 12 +- .../q86/extended.txt | 6 +- .../q86/simplified.txt | 6 +- .../q87/explain.txt | 12 +- .../q87/extended.txt | 14 +- .../q87/simplified.txt | 6 +- .../q88/explain.txt | 72 +-- .../q88/extended.txt | 64 +- .../q88/simplified.txt | 36 +- .../q89/explain.txt | 16 +- .../q89/extended.txt | 8 +- .../q89/simplified.txt | 8 +- .../q9/explain.txt | 24 +- .../q9/extended.txt | 12 +- .../q9/simplified.txt | 12 +- .../q90/explain.txt | 24 +- .../q90/extended.txt | 16 +- .../q90/simplified.txt | 12 +- .../q91/explain.txt | 28 +- .../q91/extended.txt | 14 +- .../q91/simplified.txt | 14 +- .../q92/explain.txt | 12 +- .../q92/extended.txt | 8 +- .../q92/simplified.txt | 6 +- .../q93/explain.txt | 12 +- .../q93/extended.txt | 6 +- .../q93/simplified.txt | 6 +- .../q94/explain.txt | 24 +- .../q94/extended.txt | 12 +- .../q94/simplified.txt | 12 +- .../q95/explain.txt | 24 +- .../q95/extended.txt | 18 +- .../q95/simplified.txt | 12 +- .../q96/explain.txt | 16 +- .../q96/extended.txt | 8 +- .../q96/simplified.txt | 8 +- .../q97/explain.txt | 8 +- .../q97/extended.txt | 6 +- .../q97/simplified.txt | 4 +- .../q98/explain.txt | 12 +- .../q98/extended.txt | 6 +- .../q98/simplified.txt | 6 +- .../q99/explain.txt | 20 +- .../q99/extended.txt | 10 +- .../q99/simplified.txt | 10 +- .../q10a/explain.txt | 20 +- .../q10a/extended.txt | 14 +- .../q10a/simplified.txt | 10 +- .../q11/explain.txt | 32 +- .../q11/extended.txt | 20 +- .../q11/simplified.txt | 16 +- .../q12/explain.txt | 12 +- .../q12/extended.txt | 6 +- .../q12/simplified.txt | 6 +- .../q14/explain.txt | 48 +- .../q14/extended.txt | 86 +-- .../q14/simplified.txt | 24 +- .../q14a/explain.txt | 40 +- .../q14a/extended.txt | 590 +++++++++--------- .../q14a/simplified.txt | 20 +- .../q18a/explain.txt | 44 +- .../q18a/extended.txt | 62 +- .../q18a/simplified.txt | 22 +- .../q20/explain.txt | 12 +- .../q20/extended.txt | 6 +- .../q20/simplified.txt | 6 +- .../q22/explain.txt | 16 +- .../q22/extended.txt | 8 +- .../q22/simplified.txt | 8 +- .../q22a/explain.txt | 16 +- .../q22a/extended.txt | 40 +- .../q22a/simplified.txt | 8 +- .../q24/explain.txt | 28 +- .../q24/extended.txt | 24 +- .../q24/simplified.txt | 14 +- .../q27a/explain.txt | 28 +- .../q27a/extended.txt | 26 +- .../q27a/simplified.txt | 14 +- .../q34/explain.txt | 20 +- .../q34/extended.txt | 10 +- .../q34/simplified.txt | 10 +- .../q35/explain.txt | 20 +- .../q35/extended.txt | 14 +- .../q35/simplified.txt | 10 +- .../q35a/explain.txt | 20 +- .../q35a/extended.txt | 14 +- .../q35a/simplified.txt | 10 +- .../q36a/explain.txt | 16 +- .../q36a/extended.txt | 24 +- .../q36a/simplified.txt | 8 +- .../q47/explain.txt | 16 +- .../q47/extended.txt | 24 +- .../q47/simplified.txt | 8 +- .../q49/explain.txt | 20 +- .../q49/extended.txt | 14 +- .../q49/simplified.txt | 10 +- .../q51a/explain.txt | 8 +- .../q51a/extended.txt | 24 +- .../q51a/simplified.txt | 4 +- .../q57/explain.txt | 16 +- .../q57/extended.txt | 24 +- .../q57/simplified.txt | 8 +- .../q5a/explain.txt | 24 +- .../q5a/extended.txt | 48 +- .../q5a/simplified.txt | 12 +- .../q6/explain.txt | 28 +- .../q6/extended.txt | 14 +- .../q6/simplified.txt | 14 +- .../q64/explain.txt | 68 +- .../q64/extended.txt | 76 +-- .../q64/simplified.txt | 34 +- .../q67a/explain.txt | 16 +- .../q67a/extended.txt | 72 +-- .../q67a/simplified.txt | 8 +- .../q70a/explain.txt | 16 +- .../q70a/extended.txt | 30 +- .../q70a/simplified.txt | 8 +- .../q72/explain.txt | 46 +- .../q72/extended.txt | 22 +- .../q72/simplified.txt | 22 +- .../q74/explain.txt | 32 +- .../q74/extended.txt | 20 +- .../q74/simplified.txt | 16 +- .../q75/explain.txt | 32 +- .../q75/extended.txt | 40 +- .../q75/simplified.txt | 16 +- .../q77a/explain.txt | 16 +- .../q77a/extended.txt | 66 +- .../q77a/simplified.txt | 8 +- .../q78/explain.txt | 20 +- .../q78/extended.txt | 14 +- .../q78/simplified.txt | 10 +- .../q80a/explain.txt | 40 +- .../q80a/extended.txt | 96 +-- .../q80a/simplified.txt | 20 +- .../q86a/explain.txt | 12 +- .../q86a/extended.txt | 18 +- .../q86a/simplified.txt | 6 +- .../q98/explain.txt | 12 +- .../q98/extended.txt | 6 +- .../q98/simplified.txt | 6 +- 405 files changed, 3981 insertions(+), 3981 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt index e66401996f..457920577c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt @@ -18,7 +18,7 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (25) : : +- CometFilter (24) : : +- CometHashAggregate (23) @@ -35,11 +35,11 @@ : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + : +- CometNativeScan parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + +- CometNativeScan parquet spark_catalog.default.customer (34) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -54,7 +54,7 @@ ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(29) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_county#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#4] Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(36) CometNativeScan parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) + +- CometNativeScan parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(48) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/extended.txt index a6f33d6f7e..ee476d727a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/extended.txt @@ -14,7 +14,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -24,11 +24,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -38,7 +38,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -48,16 +48,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt index e7193f87e1..a95300f752 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -30,11 +30,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -62,7 +62,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt index 091ad62567..a89cd3317e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt @@ -17,13 +17,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -34,13 +34,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (52) : +- CometFilter (51) : +- CometHashAggregate (50) @@ -52,7 +52,7 @@ : : +- CometBroadcastHashJoin (43) : : :- CometProject (39) : : : +- CometFilter (38) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : : : +- CometNativeScan parquet spark_catalog.default.customer (37) : : +- CometBroadcastExchange (42) : : +- CometFilter (41) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) @@ -67,14 +67,14 @@ : +- CometBroadcastHashJoin (61) : :- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : : +- CometNativeScan parquet spark_catalog.default.customer (55) : +- CometBroadcastExchange (60) : +- CometFilter (59) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (58) +- ReusedExchange (63) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -114,7 +114,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -156,7 +156,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -196,7 +196,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -247,7 +247,7 @@ Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight Input [5]: [customer_id#23, year_total#24, customer_id#41, customer_preferred_cust_flag#42, year_total#43] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] -(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(37) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -330,7 +330,7 @@ Arguments: [customer_id#23], [customer_id#66], Inner, BuildRight Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, customer_id#66, year_total#67] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(55) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -422,10 +422,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometFilter (75) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) + +- CometNativeScan parquet spark_catalog.default.date_dim (74) -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(74) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -447,10 +447,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometFilter (79) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (78) + +- CometNativeScan parquet spark_catalog.default.date_dim (78) -(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(78) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/extended.txt index 781eae9054..529dda10fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/extended.txt @@ -16,7 +16,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -24,10 +24,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -38,7 +38,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -46,10 +46,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -61,14 +61,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -79,13 +79,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 80 out of 86 eligible operators (93%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt index 4599ecf539..1e41f9f35e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt @@ -18,7 +18,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -28,10 +28,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -52,10 +52,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] @@ -83,7 +83,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt index 93308784a9..d2fc364869 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject (23) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt index 3f41c97ff5..282edf0ce0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt index fb83fd2f9a..7a0da92e94 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt index 74da8ba883..a85cf5ae8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt @@ -17,22 +17,22 @@ : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) +- CometBroadcastExchange (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) + +- CometNativeScan parquet spark_catalog.default.household_demographics (26) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -47,7 +47,7 @@ ReadSchema: struct= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -71,7 +71,7 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -99,7 +99,7 @@ Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(20) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -155,7 +155,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(26) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#24, hd_dep_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -203,10 +203,10 @@ BroadcastExchange (39) +- * CometColumnarToRow (38) +- CometProject (37) +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) + +- CometNativeScan parquet spark_catalog.default.date_dim (35) -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(35) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/extended.txt index 08e9beb692..a14a409b88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/extended.txt @@ -19,24 +19,24 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : +- CometNativeScan parquet spark_catalog.default.customer_demographics +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + +- CometNativeScan parquet spark_catalog.default.household_demographics Comet accelerated 36 out of 38 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt index 8ef882a435..03a7b35fad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt @@ -23,22 +23,22 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt index 22f1896b63..11e23d7574 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt @@ -22,7 +22,7 @@ : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -37,7 +37,7 @@ : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) @@ -47,11 +47,11 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -65,12 +65,12 @@ : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) :- CometProject (80) : +- CometFilter (79) : +- CometHashAggregate (78) @@ -115,7 +115,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -138,7 +138,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -161,7 +161,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -185,7 +185,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -328,7 +328,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -360,7 +360,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -684,10 +684,10 @@ BroadcastExchange (124) +- * CometColumnarToRow (123) +- CometProject (122) +- CometFilter (121) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (120) + +- CometNativeScan parquet spark_catalog.default.date_dim (120) -(120) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(120) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -714,10 +714,10 @@ BroadcastExchange (129) +- * CometColumnarToRow (128) +- CometProject (127) +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) + +- CometNativeScan parquet spark_catalog.default.date_dim (125) -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(125) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#125] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/extended.txt index 4af04a7846..cbb5339407 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/extended.txt @@ -20,7 +20,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -28,7 +28,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -36,7 +36,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -52,12 +52,12 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -74,11 +74,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -89,15 +89,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -108,20 +108,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -138,11 +138,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -153,15 +153,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -172,15 +172,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometProject : +- CometFilter : : +- ReusedSubquery @@ -199,7 +199,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -216,11 +216,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -231,15 +231,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -250,20 +250,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -280,11 +280,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -295,15 +295,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -314,15 +314,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometProject +- CometFilter : +- ReusedSubquery @@ -341,7 +341,7 @@ CometColumnarToRow : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -358,11 +358,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -373,15 +373,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -392,20 +392,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -422,11 +422,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -437,15 +437,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -456,14 +456,14 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 424 out of 458 eligible operators (92%). Final plan contains 9 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt index 79c782f2ca..a587d19fb5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt @@ -49,12 +49,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -73,11 +73,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -88,11 +88,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 CometProject [i_brand_id,i_class_id,i_category_id] @@ -107,12 +107,12 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #4 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt index 879213d892..68d1802ea5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt @@ -17,7 +17,7 @@ : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -32,7 +32,7 @@ : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) @@ -42,11 +42,11 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -60,12 +60,12 @@ : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -83,7 +83,7 @@ +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -121,7 +121,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(72) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(103) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(108) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) + +- CometNativeScan parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(112) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#103] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) + +- CometNativeScan parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(117) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) + +- CometNativeScan parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(122) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/extended.txt index f56d229b68..5011719f37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -23,7 +23,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -31,7 +31,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -51,13 +51,13 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -74,11 +74,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -89,15 +89,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -108,20 +108,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -138,11 +138,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -153,15 +153,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -172,16 +172,16 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter : +- ReusedSubquery @@ -204,13 +204,13 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -227,11 +227,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -242,15 +242,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -261,20 +261,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -291,11 +291,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -306,15 +306,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -325,15 +325,15 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter : +- ReusedSubquery - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 298 out of 327 eligible operators (91%). Final plan contains 10 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt index fb9abae378..4ae30a0feb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -75,11 +75,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -90,11 +90,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -109,13 +109,13 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt index af9b2efbd1..0003983dd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt @@ -14,15 +14,15 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : : +- CometNativeScan parquet spark_catalog.default.customer (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : +- CometNativeScan parquet spark_catalog.default.customer_address (8) +- CometBroadcastExchange (17) +- CometProject (16) +- CometFilter (15) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#5, c_current_addr_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -61,7 +61,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -89,7 +89,7 @@ Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + +- CometNativeScan parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/extended.txt index 6de0c64850..3089496e4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/extended.txt @@ -16,17 +16,17 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 26 out of 28 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt index c39b96efe3..e1b5409c8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt index f2473f1f01..2a3c8932ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) + +- CometNativeScan parquet spark_catalog.default.call_center (29) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(6) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -92,7 +92,7 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(12) CometNativeScan parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -115,7 +115,7 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(29) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt index 2ad029e444..9889eeaca3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt @@ -18,26 +18,26 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : +- CometSort : : : +- CometExchange : : : +- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + +- CometNativeScan parquet spark_catalog.default.call_center Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt index 8427aa49a9..429d83d08c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [cs_order_number] #2 CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] CometSort [cs_warehouse_sk,cs_order_number] CometExchange [cs_order_number] #3 CometProject [cs_warehouse_sk,cs_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] CometSort [cr_order_number] CometExchange [cr_order_number] #4 CometProject [cr_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cc_call_center_sk] #7 CometProject [cc_call_center_sk] CometFilter [cc_call_center_sk,cc_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt index f502a5cd8d..faa4f6694b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt @@ -29,20 +29,20 @@ : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + : +- CometNativeScan parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + +- CometNativeScan parquet spark_catalog.default.item (34) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -107,7 +107,7 @@ Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_quarter_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -135,7 +135,7 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_quarter_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -175,7 +175,7 @@ Arguments: [cs_sold_date_sk#17], [d_date_sk#23], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#23] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(28) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_state#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -203,7 +203,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#24, s_state#26] Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(34) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -259,10 +259,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_quarter_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -289,10 +289,10 @@ BroadcastExchange (54) +- * CometColumnarToRow (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) + +- CometNativeScan parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(50) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_quarter_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/extended.txt index 4e1a4f3ec0..39aa7c3908 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/extended.txt @@ -24,7 +24,7 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -33,7 +33,7 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -41,22 +41,22 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 52 out of 57 eligible operators (91%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt index 1d48f96b09..a2b650507d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] @@ -39,7 +39,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] @@ -47,17 +47,17 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] ReusedExchange [d_date_sk] #7 CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt index 5fefd21bfa..7da120375f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt @@ -22,26 +22,26 @@ : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometProject (22) : : +- CometFilter (21) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : +- CometNativeScan parquet spark_catalog.default.date_dim (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + +- CometNativeScan parquet spark_catalog.default.item (32) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -56,7 +56,7 @@ ReadSchema: struct @@ -77,7 +77,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -105,7 +105,7 @@ Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(15) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#17, c_current_addr_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -129,7 +129,7 @@ Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#19, ca_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -157,7 +157,7 @@ Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(26) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#22, s_zip#23] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/extended.txt index a8cfde2d31..fbe6798e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/extended.txt @@ -15,24 +15,24 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 35 out of 35 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt index 93ab89c142..675500cd0b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt @@ -17,22 +17,22 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_zip] #5 CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [s_store_sk,s_zip] #6 CometProject [s_zip] [s_store_sk,s_zip] CometFilter [s_store_sk,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt index 58e7d31f47..40e213818e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt @@ -13,17 +13,17 @@ : : +- CometBroadcastHashJoin (10) : : :- CometUnion (5) : : : :- CometProject (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometProject (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (3) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (3) : : +- CometBroadcastExchange (9) : : +- CometProject (8) : : +- CometFilter (7) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (6) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (6) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (15) +- CometBroadcastExchange (29) +- CometProject (28) +- CometBroadcastHashJoin (27) @@ -32,13 +32,13 @@ +- CometBroadcastExchange (26) +- CometProject (25) +- CometFilter (24) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (23) + +- CometNativeScan parquet spark_catalog.default.date_dim (23) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#2)] ReadSchema: struct @@ -46,10 +46,10 @@ ReadSchema: struct Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(3) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#6)] ReadSchema: struct @@ -61,7 +61,7 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(6) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#1 Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(15) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -139,7 +139,7 @@ Input [8]: [d_week_seq#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#4 Keys [1]: [d_week_seq#37] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(23) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#45, d_year#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/extended.txt index 15f5db847d..7e6f4d3a3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/extended.txt @@ -12,17 +12,17 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometUnion : : : :- CometProject - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin @@ -33,16 +33,16 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometUnion : : :- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometProject - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt index 852c5fca0f..e4b6e81639 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] CometUnion [sold_date_sk,sales_price] CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] @@ -33,4 +33,4 @@ WholeStageCodegen (1) CometBroadcastExchange [d_week_seq] #6 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt index c23383e09a..202b8afcda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject (23) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt index cd52b2cd12..754a4fad60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt index 2958d060fe..7d40087d4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt index 5656916223..d8a4f52824 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt @@ -15,14 +15,14 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + : +- CometNativeScan parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory @@ -37,7 +37,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -115,7 +115,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(10) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -182,7 +182,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(25) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -197,7 +197,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(28) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,7 +256,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -380,10 +380,10 @@ BroadcastExchange (72) +- * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) + +- CometNativeScan parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(68) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,10 +410,10 @@ BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometProject (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -453,7 +453,7 @@ Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquer +- CometBroadcastExchange (86) +- CometProject (85) +- CometFilter (84) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (83) + +- CometNativeScan parquet spark_catalog.default.date_dim (83) (78) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -480,7 +480,7 @@ Arguments: [ss_customer_sk#51], [c_customer_sk#56], Inner, BuildRight Input [5]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] Arguments: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56], [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] -(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(83) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#57, d_year#58] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -544,10 +544,10 @@ BroadcastExchange (100) +- * CometColumnarToRow (99) +- CometProject (98) +- CometFilter (97) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (96) + +- CometNativeScan parquet spark_catalog.default.date_dim (96) -(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(96) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#57, d_year#58] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/extended.txt index a84226b077..ad7c278937 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/extended.txt @@ -17,7 +17,7 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter @@ -35,14 +35,14 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometSort : : +- CometProject : : +- CometFilter @@ -65,14 +65,14 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -80,14 +80,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometProject +- CometBroadcastHashJoin :- CometProject @@ -115,14 +115,14 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometSort : +- CometProject : +- CometFilter @@ -134,13 +134,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 127 out of 138 eligible operators (92%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt index 8f1bddf6c0..90a188196a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -41,14 +41,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -75,12 +75,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] @@ -88,14 +88,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_quantity,ws_list_price] [sales] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt index 474a708a44..5bbe134230 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt @@ -31,10 +31,10 @@ : : : : : +- CometBroadcastExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (5) : : : : +- CometBroadcastExchange (13) : : : : +- CometFilter (12) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : +- CometSort (39) : : : +- CometProject (38) : : : +- CometFilter (37) @@ -45,17 +45,17 @@ : : : +- CometBroadcastHashJoin (32) : : : :- CometProject (28) : : : : +- CometFilter (27) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) : : : +- CometBroadcastExchange (31) : : : +- CometFilter (30) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + : : : +- CometNativeScan parquet spark_catalog.default.customer (29) : : +- CometBroadcastExchange (52) : : +- CometProject (51) : : +- CometSortMergeJoin (50) : : :- CometSort (44) : : : +- CometExchange (43) : : : +- CometFilter (42) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) + : : : +- CometNativeScan parquet spark_catalog.default.customer (41) : : +- CometSort (49) : : +- CometProject (48) : : +- CometFilter (47) @@ -64,7 +64,7 @@ : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) +- CometHashAggregate (85) +- CometExchange (84) +- CometHashAggregate (83) @@ -113,7 +113,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -141,7 +141,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -208,7 +208,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -223,7 +223,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(29) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -278,7 +278,7 @@ Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold Right output [1]: [c_customer_sk#23] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(41) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -339,7 +339,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#29], Inner, BuildRight Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#29, c_first_name#32, c_last_name#33] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -494,10 +494,10 @@ BroadcastExchange (93) +- * CometColumnarToRow (92) +- CometProject (91) +- CometFilter (90) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (89) + +- CometNativeScan parquet spark_catalog.default.date_dim (89) -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(89) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -524,10 +524,10 @@ BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometProject (96) +- CometFilter (95) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + +- CometNativeScan parquet spark_catalog.default.date_dim (94) -(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(94) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -567,7 +567,7 @@ Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquer +- CometBroadcastExchange (107) +- CometProject (106) +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (104) (99) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -594,7 +594,7 @@ Arguments: [ss_customer_sk#60], [c_customer_sk#65], Inner, BuildRight Input [5]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] Arguments: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65], [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(104) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#66, d_year#67] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -658,10 +658,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) + +- CometNativeScan parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(117) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#66, d_year#67] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/extended.txt index cd91f5de49..47d6a66c8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/extended.txt @@ -20,7 +20,7 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter @@ -38,14 +38,14 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometSort : : : +- CometProject : : : +- CometFilter @@ -68,14 +68,14 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometHashAggregate : : : +- CometExchange : : : +- CometHashAggregate @@ -83,17 +83,17 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometProject : : +- CometSortMergeJoin : : :- CometSort : : : +- CometExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometSort : : +- CometProject : : +- CometFilter @@ -105,14 +105,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -145,14 +145,14 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometSort : : +- CometProject : : +- CometFilter @@ -164,17 +164,17 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometProject : +- CometSortMergeJoin : :- CometSort : : +- CometExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometSort : +- CometProject : +- CometFilter @@ -186,13 +186,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 177 out of 190 eligible operators (93%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt index 92563114ae..94b60764fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt @@ -24,7 +24,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -44,14 +44,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -78,12 +78,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] @@ -91,17 +91,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] CometExchange [c_customer_sk] #16 CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -111,7 +111,7 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #17 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] CometExchange [c_last_name,c_first_name] #18 CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt index c6e3a30a1f..67a43617c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : : +- CometNativeScan parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + : +- CometNativeScan parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(19) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(32) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : : +- CometNativeScan parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(57) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/extended.txt index 8777e81a37..b65f56f327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/extended.txt @@ -23,29 +23,29 @@ Filter : : : : : : +- CometExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : +- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item +: : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +: : +- CometNativeScan parquet spark_catalog.default.customer : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter -: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +: +- CometNativeScan parquet spark_catalog.default.customer_address +- HashAggregate +- CometColumnarToRow +- CometColumnarExchange @@ -69,28 +69,28 @@ Filter : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + +- CometNativeScan parquet spark_catalog.default.customer_address Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt index 440f477d0f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt index a77a3f88b3..65c5d9d521 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : : +- CometNativeScan parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + : +- CometNativeScan parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(19) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(32) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : : +- CometNativeScan parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(57) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/extended.txt index 8777e81a37..b65f56f327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/extended.txt @@ -23,29 +23,29 @@ Filter : : : : : : +- CometExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : +- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item +: : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +: : +- CometNativeScan parquet spark_catalog.default.customer : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter -: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +: +- CometNativeScan parquet spark_catalog.default.customer_address +- HashAggregate +- CometColumnarToRow +- CometColumnarExchange @@ -69,28 +69,28 @@ Filter : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + +- CometNativeScan parquet spark_catalog.default.customer_address Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt index 440f477d0f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt index 6a9f9094d3..4e05ac52eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt @@ -29,20 +29,20 @@ : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + : +- CometNativeScan parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + +- CometNativeScan parquet spark_catalog.default.item (34) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -107,7 +107,7 @@ Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -135,7 +135,7 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#19] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#22, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -175,7 +175,7 @@ Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#25] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(28) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -203,7 +203,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#26], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_sk#26, s_store_id#29, s_store_name#28] Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(34) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -259,10 +259,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -289,10 +289,10 @@ BroadcastExchange (54) +- * CometColumnarToRow (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) + +- CometNativeScan parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(50) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#22, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/extended.txt index 4e1a4f3ec0..39aa7c3908 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/extended.txt @@ -24,7 +24,7 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -33,7 +33,7 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -41,22 +41,22 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 52 out of 57 eligible operators (91%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt index dcad304452..380cb2db83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #3 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] @@ -39,7 +39,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #5 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] @@ -47,17 +47,17 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] ReusedExchange [d_date_sk] #7 CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #8 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt index 3d70460689..180fc4396f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt @@ -17,19 +17,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometNativeScan parquet spark_catalog.default.item (15) +- CometBroadcastExchange (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + +- CometNativeScan parquet spark_catalog.default.promotion (21) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -44,7 +44,7 @@ ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt index 7bdad4b94f..c19e6caf52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt index 4c7261693d..ff1e2b0435 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt @@ -20,11 +20,11 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : : +- CometBroadcastExchange (34) : : +- CometFilter (33) : : +- CometHashAggregate (32) @@ -44,11 +44,11 @@ : +- CometBroadcastExchange (40) : +- CometProject (39) : +- CometFilter (38) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : +- CometNativeScan parquet spark_catalog.default.customer (37) +- CometBroadcastExchange (46) +- CometProject (45) +- CometFilter (44) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns @@ -63,7 +63,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_county#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -177,7 +177,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] Condition : isnotnull(ss_addr_sk#11) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(18) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -248,7 +248,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_addr_sk#23) -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(33) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -513,10 +513,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dyn BroadcastExchange (94) +- * CometColumnarToRow (93) +- CometFilter (92) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (91) + +- CometNativeScan parquet spark_catalog.default.date_dim (91) -(91) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(91) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -538,10 +538,10 @@ Subquery:2 Hosting operator id = 16 Hosting Expression = ss_sold_date_sk#13 IN d BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometFilter (96) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (95) + +- CometNativeScan parquet spark_catalog.default.date_dim (95) -(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(95) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -563,10 +563,10 @@ Subquery:3 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN d BroadcastExchange (102) +- * CometColumnarToRow (101) +- CometFilter (100) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (99) + +- CometNativeScan parquet spark_catalog.default.date_dim (99) -(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(99) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/extended.txt index b4e7aa4db6..4979461c17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/extended.txt @@ -22,13 +22,13 @@ CometColumnarToRow : : : : : : : +- BroadcastExchange : : : : : : : +- CometColumnarToRow : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : : +- CometBroadcastExchange : : : : +- CometHashAggregate : : : : +- CometExchange @@ -43,13 +43,13 @@ CometColumnarToRow : : : : : : +- BroadcastExchange : : : : : : +- CometColumnarToRow : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometHashAggregate : : : +- CometExchange @@ -64,13 +64,13 @@ CometColumnarToRow : : : : : +- BroadcastExchange : : : : : +- CometColumnarToRow : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -84,10 +84,10 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometHashAggregate : +- CometExchange @@ -101,10 +101,10 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -118,9 +118,9 @@ CometColumnarToRow : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + +- CometNativeScan parquet spark_catalog.default.customer_address Comet accelerated 111 out of 120 eligible operators (92%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt index 2f97384320..6d5f88db6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt @@ -26,13 +26,13 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [ca_address_sk,ca_county] #5 CometFilter [ca_address_sk,ca_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #6 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #7 @@ -49,10 +49,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,store_sales] #10 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] @@ -70,10 +70,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #14 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt index d6c10008ec..1f8eec0c3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt @@ -14,7 +14,7 @@ : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometNativeScan parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -27,7 +27,7 @@ : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Condition : isnotnull(cs_item_sk#7) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + +- CometNativeScan parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/extended.txt index 943bb147ff..b3c1a352cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/extended.txt @@ -15,11 +15,11 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -33,10 +33,10 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 35 out of 38 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt index 07619fc999..6a8be191c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] @@ -37,5 +37,5 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt index a178860ccc..3d8dff6eae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt @@ -19,19 +19,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : : +- CometNativeScan parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + : +- CometNativeScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) : +- CometExchange (40) : +- CometHashAggregate (39) @@ -74,7 +74,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_manufact_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_category#13, i_manufact_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -348,10 +348,10 @@ BroadcastExchange (66) +- * CometColumnarToRow (65) +- CometProject (64) +- CometFilter (63) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (62) + +- CometNativeScan parquet spark_catalog.default.date_dim (62) -(62) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(62) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/extended.txt index 063fdb6faf..e1b4b53ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/extended.txt @@ -20,23 +20,23 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -52,19 +52,19 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -80,18 +80,18 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 89 out of 93 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt index b4b84c2ac6..2a4275f967 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt @@ -24,23 +24,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_manufact_id] #6 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #7 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_manufact_id] #8 CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt index 287e323952..ce3182de0d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt @@ -19,19 +19,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- CometNativeScan parquet spark_catalog.default.customer (25) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -46,7 +46,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/extended.txt index 3d9d23d2cc..20468961d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/extended.txt @@ -20,22 +20,22 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : +- CometNativeScan parquet spark_catalog.default.household_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 35 out of 37 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt index 47d0e35dd2..6e56af37e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt index fedc50de1e..1ab4bf7f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt @@ -15,7 +15,7 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) @@ -23,7 +23,7 @@ TakeOrderedAndProject (47) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) @@ -40,15 +40,15 @@ TakeOrderedAndProject (47) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -66,7 +66,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(29) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#22] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(36) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) + +- CometNativeScan parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(48) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/extended.txt index a6f33d6f7e..ee476d727a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/extended.txt @@ -14,7 +14,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -24,11 +24,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -38,7 +38,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -48,16 +48,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt index f1fe09fb46..5479332ae0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt index ae1d4c242b..c5a32eaa51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt @@ -20,15 +20,15 @@ TakeOrderedAndProject (30) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometNativeScan parquet spark_catalog.default.item (9) +- CometBroadcastExchange (18) +- CometProject (17) +- CometFilter (16) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometNativeScan parquet spark_catalog.default.store (15) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -43,7 +43,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -89,7 +89,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -276,10 +276,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/extended.txt index 55af64efc6..e408ea6536 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/extended.txt @@ -19,15 +19,15 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometHashAggregate : +- CometExchange @@ -42,11 +42,11 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -61,10 +61,10 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 62 out of 66 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt index 7ceccf5d2c..43062e2323 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt @@ -23,15 +23,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometExchange [c_last_name,c_first_name,d_date] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt index ed74780c2c..1b34e155ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt @@ -18,14 +18,14 @@ : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -45,7 +45,7 @@ +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory @@ -60,7 +60,7 @@ ReadSchema: struct 0.000000)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -230,7 +230,7 @@ Arguments: [c_customer_sk#28], [ss_customer_sk#36], Inner, BuildRight Input [14]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -277,7 +277,7 @@ Left output [2]: [customer_id#26, year_total#27] Right output [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] Arguments: [customer_id#26], [customer_id#47], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -360,7 +360,7 @@ Arguments: [customer_id#26], [customer_id#80], Inner, BuildRight Input [12]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, customer_id#80, year_total#81] Arguments: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81], [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -439,7 +439,7 @@ Arguments: [customer_id#26], [customer_id#101], Inner, (CASE WHEN (year_total#81 Input [13]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81, customer_id#101, year_total#102] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(71) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -522,7 +522,7 @@ Arguments: [customer_id#26], [customer_id#128], Inner, BuildRight Input [12]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, customer_id#128, year_total#129] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(89) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -614,10 +614,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dy BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(108) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -639,10 +639,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#41 IN d BroadcastExchange (115) +- * CometColumnarToRow (114) +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) + +- CometNativeScan parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(112) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/extended.txt index 1be0816ee1..c4bd842dd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/extended.txt @@ -19,7 +19,7 @@ CometColumnarToRow : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -27,10 +27,10 @@ CometColumnarToRow : : : : : : +- BroadcastExchange : : : : : : +- CometColumnarToRow : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometHashAggregate : : : : +- CometExchange @@ -41,7 +41,7 @@ CometColumnarToRow : : : : : +- CometBroadcastHashJoin : : : : : :- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -49,10 +49,10 @@ CometColumnarToRow : : : : : +- BroadcastExchange : : : : : +- CometColumnarToRow : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometHashAggregate @@ -64,14 +64,14 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -82,14 +82,14 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -101,14 +101,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -119,13 +119,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 118 out of 126 eligible operators (93%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt index 36be965b48..cd7e079da8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] @@ -31,10 +31,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 @@ -45,7 +45,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] @@ -55,10 +55,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] @@ -70,7 +70,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] @@ -86,7 +86,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] @@ -103,7 +103,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] @@ -119,7 +119,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt index 00e2b8fbc1..6a9f237f6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt @@ -20,18 +20,18 @@ : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (5) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) + : +- CometNativeScan parquet spark_catalog.default.item (18) +- CometBroadcastExchange (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -54,7 +54,7 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(5) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -86,7 +86,7 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(12) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#11, w_state#12] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -114,7 +114,7 @@ Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(18) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -142,7 +142,7 @@ Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -193,10 +193,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dyn BroadcastExchange (37) +- * CometColumnarToRow (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/extended.txt index 54a7b21486..beef88a3fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/extended.txt @@ -19,22 +19,22 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometSort : : : +- CometExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + : : +- CometNativeScan parquet spark_catalog.default.warehouse : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 34 out of 36 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt index 79e3556750..7e7073d139 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt @@ -23,20 +23,20 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] CometExchange [cr_order_number,cr_item_sk] #4 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometBroadcastExchange [w_warehouse_sk,w_state] #5 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #7 CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt index 568da58fa0..625df9cd57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : +- CometNativeScan parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,10 +17,10 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) + +- CometNativeScan parquet spark_catalog.default.item (4) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -35,7 +35,7 @@ Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_ma Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(4) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/extended.txt index 9d802b5033..b40ebddb6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/extended.txt @@ -7,7 +7,7 @@ CometColumnarToRow +- CometBroadcastHashJoin :- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter @@ -16,6 +16,6 @@ CometColumnarToRow +- CometHashAggregate +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt index 99c935aaa4..2557f51684 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] @@ -18,4 +18,4 @@ WholeStageCodegen (1) CometHashAggregate [i_manufact,count] CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt index 183fffdf00..af01446009 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt index 839937d40c..96ab34454d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 CometProject [i_category] [i_item_sk,i_category_id,i_category] CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt index 62ddd93781..261e6ddbaa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometNativeScan parquet spark_catalog.default.store (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/extended.txt index cb125e2456..5eaea983af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt index 59ad7611a5..c7064b3ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [d_day_name] [d_date_sk,d_day_name] CometFilter [d_date_sk,d_year,d_day_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt index 4b30c7d51f..b2a660d9d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt @@ -23,7 +23,7 @@ TakeOrderedAndProject (45) : : : +- CometHashAggregate (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- * Sort (32) : : +- * Project (31) : : +- * Filter (30) @@ -42,11 +42,11 @@ TakeOrderedAndProject (45) : +- * CometColumnarToRow (38) : +- CometProject (37) : +- CometFilter (36) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (35) + : +- CometNativeScan parquet spark_catalog.default.item (35) +- ReusedExchange (42) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -185,7 +185,7 @@ Join condition: None Output [3]: [item_sk#7, rnk#11, item_sk#16] Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(35) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_product_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -243,10 +243,10 @@ Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (46) + +- CometNativeScan parquet spark_catalog.default.store_sales (46) -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(46) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/extended.txt index 6b58597252..236d51688a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/extended.txt @@ -24,13 +24,13 @@ TakeOrderedAndProject : : : : +- CometHashAggregate : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometHashAggregate : : : +- CometExchange : : : +- CometHashAggregate : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- Sort : : +- Project : : +- Filter @@ -49,16 +49,16 @@ TakeOrderedAndProject : : +- CometHashAggregate : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 32 out of 55 eligible operators (58%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt index d0c3ad435b..a6db5c3397 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt @@ -34,13 +34,13 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] CometProject [ss_store_sk,ss_net_profit] CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] CometExchange [ss_item_sk] #2 CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] CometProject [ss_item_sk,ss_net_profit] CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (6) Sort [rnk] @@ -70,6 +70,6 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] InputAdapter CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt index f6ab9a8d53..9c15803e55 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt @@ -20,24 +20,24 @@ TakeOrderedAndProject (39) : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + : +- CometNativeScan parquet spark_catalog.default.item (20) +- BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) + +- CometNativeScan parquet spark_catalog.default.item (27) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -52,7 +52,7 @@ ReadSchema: struct= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -66,7 +66,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(8) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -94,7 +94,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(14) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -122,7 +122,7 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH, Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(20) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -174,10 +174,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/extended.txt index 57fbc2d035..fe50450ec6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/extended.txt @@ -17,21 +17,21 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 31 out of 33 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt index e29e101c7e..0f5cdba34b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt @@ -21,19 +21,19 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt index 338f5f4201..260bd1c80b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt @@ -26,11 +26,11 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometProject (7) : : +- CometFilter (6) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) :- * Project (49) : +- * Filter (48) : +- Window (47) @@ -52,7 +52,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) : : +- CometProject (33) : : +- CometFilter (32) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) : +- ReusedExchange (36) +- * Project (72) +- * Filter (71) @@ -75,7 +75,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) : +- CometProject (56) : +- CometFilter (55) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) +- ReusedExchange (59) @@ -99,7 +99,7 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(5) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -123,7 +123,7 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(10) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -216,7 +216,7 @@ Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, c Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(31) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -317,7 +317,7 @@ Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(54) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -430,10 +430,10 @@ BroadcastExchange (83) +- * CometColumnarToRow (82) +- CometProject (81) +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) + +- CometNativeScan parquet spark_catalog.default.date_dim (79) -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(79) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt index 75684a9669..e8140d659e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt @@ -28,14 +28,14 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : +- CometNativeScan parquet spark_catalog.default.web_returns : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- Project : +- Filter : +- Window @@ -58,11 +58,11 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- Project +- Filter +- Window @@ -85,10 +85,10 @@ CometColumnarToRow : : +- ReusedSubquery : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : +- CometNativeScan parquet spark_catalog.default.store_returns +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 66 out of 87 eligible operators (75%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt index ca80833ee5..d91eb10d33 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt @@ -40,14 +40,14 @@ WholeStageCodegen (11) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] WholeStageCodegen (6) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -76,7 +76,7 @@ WholeStageCodegen (11) ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] @@ -106,5 +106,5 @@ WholeStageCodegen (11) ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt index b4a1dedd61..7dfc78495f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt @@ -23,11 +23,11 @@ : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- CometNativeScan parquet spark_catalog.default.store (14) :- CometHashAggregate (41) : +- CometExchange (40) : +- CometHashAggregate (39) @@ -46,7 +46,7 @@ : +- CometBroadcastExchange (36) : +- CometProject (35) : +- CometFilter (34) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (33) + : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) +- CometHashAggregate (64) +- CometExchange (63) +- CometHashAggregate (62) @@ -64,12 +64,12 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (45) : : +- CometProject (49) : : +- CometFilter (48) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) : +- ReusedExchange (53) +- CometBroadcastExchange (59) +- CometProject (58) +- CometFilter (57) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (56) + +- CometNativeScan parquet spark_catalog.default.web_site (56) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -108,7 +108,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,7 +136,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -226,7 +226,7 @@ Arguments: [date_sk#37], [d_date_sk#52], Inner, BuildRight Input [7]: [page_sk#36, date_sk#37, sales_price#38, profit#39, return_amt#40, net_loss#41, d_date_sk#52] Arguments: [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41], [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41] -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +(33) CometNativeScan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -295,7 +295,7 @@ ReadSchema: struct @@ -67,7 +67,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_numb Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(8) CometNativeScan parquet spark_catalog.default.store Output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -95,7 +95,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [1]: [d_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -119,7 +119,7 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#26], Inner, BuildRight Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#26] Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -175,10 +175,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/extended.txt index 0fbba7d1e8..935b0a2e1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/extended.txt @@ -12,7 +12,7 @@ CometColumnarToRow : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns @@ -21,17 +21,17 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 31 out of 33 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt index 98a44eaf40..5eddc939fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] @@ -25,15 +25,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #4 CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastExchange [d_date_sk] #5 CometFilter [d_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt index 683cc25d24..be8e4fd945 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt @@ -24,7 +24,7 @@ TakeOrderedAndProject (41) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) +- CometSort (33) +- CometColumnarExchange (32) +- * Project (31) @@ -54,7 +54,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -227,10 +227,10 @@ BroadcastExchange (46) +- * CometColumnarToRow (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) + +- CometNativeScan parquet spark_catalog.default.date_dim (42) -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(42) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt index 66c5717cc7..2465344e99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt @@ -25,11 +25,11 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometSort +- CometColumnarExchange +- Project @@ -48,6 +48,6 @@ TakeOrderedAndProject +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 37 out of 47 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt index a5af2e114b..b8034c5f59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt @@ -35,11 +35,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] CometColumnarExchange [item_sk,d_date] #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt index a05651a422..4af1b4f562 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt index 218062c3ef..cee223d027 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt index 2fc38bf838..d77e8c3b09 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometNativeScan parquet spark_catalog.default.store (15) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt index 825b1ed81c..77b01e59ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -25,13 +25,13 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 27 out of 33 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt index 0c123eb728..827a521610 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manufact_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] @@ -31,11 +31,11 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [d_date_sk,d_qoy] #5 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [s_store_sk] #6 CometFilter [s_store_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt index 26ce2a5731..8e37cf65c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt @@ -34,29 +34,29 @@ : : : : : : +- CometBroadcastExchange (11) : : : : : : +- CometProject (10) : : : : : : +- CometFilter (9) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (8) : : : : : +- CometBroadcastExchange (17) : : : : : +- CometProject (16) : : : : : +- CometFilter (15) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) : : : : +- CometBroadcastExchange (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (30) : : : +- CometFilter (29) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) : : +- CometBroadcastExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (33) : +- CometBroadcastExchange (42) : +- CometProject (41) : +- CometFilter (40) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + : +- CometNativeScan parquet spark_catalog.default.store (39) +- CometBroadcastExchange (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -95,7 +95,7 @@ Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(8) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_class#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -123,7 +123,7 @@ Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] Arguments: [customer_sk#6], [customer_sk#6] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(20) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#20, c_current_addr_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -214,7 +214,7 @@ Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(33) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -242,7 +242,7 @@ Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(39) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_county#30, s_state#31] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -270,7 +270,7 @@ Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildR Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -340,10 +340,10 @@ BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometProject (61) +- CometFilter (60) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) + +- CometNativeScan parquet spark_catalog.default.date_dim (59) -(59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(59) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -372,10 +372,10 @@ BroadcastExchange (68) +- * CometColumnarToRow (67) +- CometProject (66) +- CometFilter (65) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) + +- CometNativeScan parquet spark_catalog.default.date_dim (64) -(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(64) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -404,10 +404,10 @@ Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) + +- CometNativeScan parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(69) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#44, d_year#45, d_moy#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -446,10 +446,10 @@ Subquery:5 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (79) +- CometProject (78) +- CometFilter (77) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) + +- CometNativeScan parquet spark_catalog.default.date_dim (76) -(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(76) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#48, d_year#49, d_moy#50] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/extended.txt index 98fce92fb1..87af0bb5d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/extended.txt @@ -32,7 +32,7 @@ CometColumnarToRow : : : : : : : : +- CometColumnarToRow : : : : : : : : +- CometProject : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : +- CometProject : : : : : : : +- CometFilter : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -40,14 +40,14 @@ CometColumnarToRow : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -63,7 +63,7 @@ CometColumnarToRow : : : : : +- CometHashAggregate : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- Subquery : : : : +- CometColumnarToRow : : : : +- CometHashAggregate @@ -71,21 +71,21 @@ CometColumnarToRow : : : : +- CometHashAggregate : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter : :- ReusedSubquery : +- ReusedSubquery - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 75 out of 84 eligible operators (89%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt index ae0cecb1ba..4f3529bb52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt @@ -36,7 +36,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] @@ -44,14 +44,14 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk] #5 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] @@ -71,7 +71,7 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #4 WholeStageCodegen (1) CometColumnarToRow @@ -81,19 +81,19 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] CometBroadcastExchange [s_county,s_state] #13 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_county,s_state] CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [(d_month_seq + 1)] #3 ReusedSubquery [(d_month_seq + 3)] #4 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt index 47e0117481..358eb94c69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,10 +35,10 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/extended.txt index a292badf5d..e09de16f3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/extended.txt @@ -9,13 +9,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : +- CometNativeScan parquet spark_catalog.default.store_sales +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt index dd7658cbac..a2adacc531 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt index 5562e86b91..a0c0ffb8dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt @@ -19,20 +19,20 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : : +- CometNativeScan parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + : +- CometNativeScan parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_color#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) + +- CometNativeScan parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(63) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/extended.txt index 36b9a8306d..0236f37402 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/extended.txt @@ -20,24 +20,24 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -53,20 +53,20 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -82,19 +82,19 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 92 out of 96 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt index 19eca782cd..350da49c9f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt index eb21a3abd7..df441af4f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(14) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt index de7d26cd06..3ab8670a2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -30,13 +30,13 @@ TakeOrderedAndProject : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : : +- CometNativeScan parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -54,7 +54,7 @@ TakeOrderedAndProject : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -62,13 +62,13 @@ TakeOrderedAndProject : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : +- CometNativeScan parquet spark_catalog.default.call_center +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -86,7 +86,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -94,12 +94,12 @@ TakeOrderedAndProject : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + +- CometNativeScan parquet spark_catalog.default.call_center Comet accelerated 75 out of 97 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt index b854e818db..3bfe35a7e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] @@ -38,13 +38,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt index 0307d0df0e..597d0cf23e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt @@ -18,12 +18,12 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- CometBroadcastExchange (33) : +- CometFilter (32) @@ -65,7 +65,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -93,7 +93,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -296,14 +296,14 @@ BroadcastExchange (62) +- CometProject (60) +- CometBroadcastHashJoin (59) :- CometFilter (54) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) + : +- CometNativeScan parquet spark_catalog.default.date_dim (53) +- CometBroadcastExchange (58) +- CometProject (57) +- CometFilter (56) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometNativeScan parquet spark_catalog.default.date_dim (55) -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(53) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -314,7 +314,7 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#10, d_week_seq#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -353,10 +353,10 @@ Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquer * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) + +- CometNativeScan parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(63) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#41, d_week_seq#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/extended.txt index 3d3dd68990..56b820ff92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/extended.txt @@ -20,7 +20,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter @@ -28,17 +28,17 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter @@ -46,8 +46,8 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -63,12 +63,12 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter @@ -76,8 +76,8 @@ CometColumnarToRow : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -93,12 +93,12 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter @@ -106,7 +106,7 @@ CometColumnarToRow : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 96 out of 104 eligible operators (92%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt index 96e11bc0c7..b019b528c2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt @@ -24,7 +24,7 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] CometFilter [d_date,d_week_seq] @@ -34,17 +34,17 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cs_item_rev] #6 CometFilter [item_id,cs_item_rev] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt index 4366c901e1..1bb4b7dc8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt @@ -13,19 +13,19 @@ : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (12) + : : +- CometNativeScan parquet spark_catalog.default.store (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) + : +- CometNativeScan parquet spark_catalog.default.date_dim (18) +- CometBroadcastExchange (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -36,17 +36,17 @@ : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) + : +- CometNativeScan parquet spark_catalog.default.store (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -55,7 +55,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -97,7 +97,7 @@ Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, s Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(12) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -125,7 +125,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(18) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -161,7 +161,7 @@ Input [9]: [d_week_seq#38, ss_store_sk#39, sum#40, sum#41, sum#42, sum#43, sum#4 Keys [2]: [d_week_seq#38, ss_store_sk#39] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#47 END))] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(26) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#48, s_store_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -189,7 +189,7 @@ Arguments: [ss_store_sk#39], [s_store_sk#48], Inner, BuildRight Input [11]: [d_week_seq#38, ss_store_sk#39, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_sk#48, s_store_id#50] Arguments: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50], [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(32) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#58, d_week_seq#59] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/extended.txt index 3d50a2131e..bfd467b72b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/extended.txt @@ -12,19 +12,19 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin @@ -36,18 +36,18 @@ CometColumnarToRow : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt index bca9c6463d..72823b14c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] @@ -37,8 +37,8 @@ WholeStageCodegen (1) CometBroadcastExchange [s_store_sk,s_store_id] #6 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [d_week_seq] #7 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt index da1b420416..640932ef8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt @@ -15,22 +15,22 @@ : : : +- CometBroadcastHashJoin (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (1) : : : +- CometBroadcastExchange (6) : : : +- CometFilter (5) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : : +- CometNativeScan parquet spark_catalog.default.customer (4) : : +- CometBroadcastExchange (11) : : +- CometFilter (10) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) +- CometBroadcastExchange (32) +- CometProject (31) +- CometBroadcastHashJoin (30) :- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + : +- CometNativeScan parquet spark_catalog.default.item (20) +- CometBroadcastExchange (29) +- CometFilter (28) +- CometHashAggregate (27) @@ -38,10 +38,10 @@ +- CometHashAggregate (25) +- CometProject (24) +- CometFilter (23) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + +- CometNativeScan parquet spark_catalog.default.item (22) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(1) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -56,7 +56,7 @@ Condition : isnotnull(ca_address_sk#1) Input [2]: [ca_address_sk#1, ca_state#2] Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(4) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -105,7 +105,7 @@ Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -133,7 +133,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(20) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(22) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_current_price#17, i_category#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) + +- CometNativeScan parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(46) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/extended.txt index f5b69fc6cf..5cce29dd89 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/extended.txt @@ -14,10 +14,10 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -33,18 +33,18 @@ CometColumnarToRow : : : +- CometHashAggregate : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -52,6 +52,6 @@ CometColumnarToRow +- CometHashAggregate +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt index 2c772b5998..7318a7bfe7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt @@ -16,10 +16,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt index e5e165b5ec..535ea603da 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt @@ -19,20 +19,20 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : : +- CometNativeScan parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + : +- CometNativeScan parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_category#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) + +- CometNativeScan parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(63) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/extended.txt index 36b9a8306d..0236f37402 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/extended.txt @@ -20,24 +20,24 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -53,20 +53,20 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -82,19 +82,19 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 92 out of 96 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt index 086da3b776..45ea4b263e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt index 4f090653c8..b1f35c55a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt @@ -22,26 +22,26 @@ : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer (21) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + : +- CometNativeScan parquet spark_catalog.default.item (32) +- BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometHashAggregate (61) @@ -78,7 +78,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -123,7 +123,7 @@ Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(18) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/extended.txt index 1fc09bd2e3..6825c99831 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/extended.txt @@ -7,7 +7,7 @@ CometColumnarToRow : :- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometFilter : : +- CometHashAggregate @@ -22,15 +22,15 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -47,6 +47,6 @@ CometColumnarToRow +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 45 out of 48 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt index cd1c84b6f2..f303607d4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] CometFilter [s_store_sk,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] CometHashAggregate [sum] [ss_store_sk,ss_item_sk,revenue,sum(UnscaledValue(ss_sales_price))] @@ -26,15 +26,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #5 CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometBroadcastExchange [ss_store_sk,ave] #6 CometFilter [ss_store_sk,ave] CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt index 7599e9ecd3..d6ed31b8e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt @@ -21,18 +21,18 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) + : +- CometNativeScan parquet spark_catalog.default.ship_mode (20) +- CometHashAggregate (45) +- CometExchange (44) +- CometHashAggregate (43) @@ -64,7 +64,7 @@ ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -173,7 +173,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(28) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -205,7 +205,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#2] Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#18] -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(35) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -269,10 +269,10 @@ BroadcastExchange (51) +- * CometColumnarToRow (50) +- CometProject (49) +- CometFilter (48) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + +- CometNativeScan parquet spark_catalog.default.date_dim (47) -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(47) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/extended.txt index b4fbb455ef..738d55c552 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/extended.txt @@ -13,7 +13,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -23,11 +23,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -37,7 +37,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -47,16 +47,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 53 eligible operators (66%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt index 68bf32c40b..56afb9227a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt @@ -17,7 +17,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -29,11 +29,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -61,7 +61,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -69,4 +69,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt index 0e32283618..651a473d07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt @@ -17,19 +17,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometNativeScan parquet spark_catalog.default.item (15) +- CometBroadcastExchange (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + +- CometNativeScan parquet spark_catalog.default.promotion (21) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -44,7 +44,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -93,7 +93,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -119,7 +119,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -279,10 +279,10 @@ BroadcastExchange (54) +- * CometColumnarToRow (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) + +- CometNativeScan parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(50) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt index 45a2c7a669..5e305b7021 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt @@ -21,17 +21,17 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- BroadcastHashJoin :- CometColumnarToRow : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- BroadcastExchange +- Project +- Filter @@ -52,10 +52,10 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 34 out of 53 eligible operators (64%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt index 80f0cfc8c2..052062f542 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt @@ -31,11 +31,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -44,7 +44,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #6 WholeStageCodegen (2) @@ -70,5 +70,5 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt index bb217d6417..3514998a69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt @@ -12,7 +12,7 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometNativeScan parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) @@ -21,7 +21,7 @@ : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) @@ -35,10 +35,10 @@ +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) + +- CometNativeScan parquet spark_catalog.default.time_dim (26) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -69,7 +69,7 @@ ReadSchema: struct @@ -102,7 +102,7 @@ Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < c Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -126,7 +126,7 @@ Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#16, i_item_desc#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -150,7 +150,7 @@ Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#18, cd_marital_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -178,7 +178,7 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#20, hd_buy_potential#21] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -206,7 +206,7 @@ Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -234,7 +234,7 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -258,7 +258,7 @@ Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#28, d_date#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -282,7 +282,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) + +- CometNativeScan parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(65) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/extended.txt index 9000d27d1b..4f2941b209 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/extended.txt @@ -32,41 +32,41 @@ CometColumnarToRow : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : +- CometProject : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion +- CometSort +- CometExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + +- CometNativeScan parquet spark_catalog.default.catalog_returns Comet accelerated 66 out of 68 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt index 678f8c67df..e7af25252a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt index 346e110371..6c24729ac4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt @@ -19,19 +19,19 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- CometNativeScan parquet spark_catalog.default.customer (25) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -46,7 +46,7 @@ ReadSchema: struct= 1) AND (cnt#17 <= 5)) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/extended.txt index 3d9d23d2cc..20468961d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/extended.txt @@ -20,22 +20,22 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : +- CometNativeScan parquet spark_catalog.default.household_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 35 out of 37 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt index 227dd00f2b..1560bcd1ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt index 649b5dc4d5..0e75a85787 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,7 +51,7 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/extended.txt index 476c7be954..18c1c90d20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -23,10 +23,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -37,7 +37,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -45,10 +45,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -60,14 +60,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -78,13 +78,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 79 out of 85 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt index 72e39422e0..6411ab4b43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -27,10 +27,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -51,10 +51,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] @@ -66,7 +66,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt index 7afb1e3be6..e3a8208f70 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt @@ -26,15 +26,15 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -51,7 +51,7 @@ : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -68,7 +68,7 @@ : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -92,7 +92,7 @@ : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -137,7 +137,7 @@ ReadSchema: struct @@ -48,7 +48,7 @@ ReadSchema: struct @@ -136,10 +136,10 @@ Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(22) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#29)] PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/extended.txt index b56a6590db..cbb627c528 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/extended.txt @@ -9,39 +9,39 @@ CometColumnarToRow : :- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometProject : +- CometBroadcastHashJoin : :- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometProject +- CometBroadcastHashJoin :- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 44 out of 44 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt index b5449831cb..abb482c3d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt @@ -11,20 +11,20 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category] #2 CometProject [i_category] [i_item_sk,i_category] CometFilter [i_item_sk,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] @@ -32,6 +32,6 @@ WholeStageCodegen (1) CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt index bb7ed0a50c..0a9506ad22 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt @@ -21,10 +21,10 @@ TakeOrderedAndProject (85) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (28) : +- CometHashAggregate (27) : +- CometExchange (26) @@ -71,7 +71,7 @@ TakeOrderedAndProject (85) : : +- ReusedExchange (53) : +- CometBroadcastExchange (58) : +- CometFilter (57) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (56) + : +- CometNativeScan parquet spark_catalog.default.web_page (56) +- CometBroadcastExchange (75) +- CometHashAggregate (74) +- CometExchange (73) @@ -98,7 +98,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -85,7 +85,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -113,7 +113,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(15) CometNativeScan parquet spark_catalog.default.customer_address Output [1]: [ca_zip#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -127,7 +127,7 @@ Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVa Input [1]: [ca_zip#12] Arguments: [ca_zip#13], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) AS ca_zip#13] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(18) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#14, ca_zip#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -142,7 +142,7 @@ Condition : isnotnull(ca_address_sk#14) Input [2]: [ca_address_sk#14, ca_zip#15] Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#15, 10, true, false, true) AS ca_zip#16] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(21) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,10 +256,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(45) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/extended.txt index 8e04bbeba7..74d9e64d61 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/extended.txt @@ -16,15 +16,15 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -32,7 +32,7 @@ CometColumnarToRow +- CometBroadcastHashJoin :- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter @@ -43,10 +43,10 @@ CometColumnarToRow +- CometBroadcastHashJoin :- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 46 out of 48 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt index d1542ab607..f20d472e8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometExchange [ca_zip] #6 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_zip] CometBroadcastExchange [ca_zip] #7 CometProject [ca_zip] CometFilter [ca_zip,cnt] @@ -47,8 +47,8 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [c_current_addr_sk] #9 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt index f0a163bf69..c3a57562e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt @@ -27,23 +27,23 @@ : : : : : +- CometExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) : : : : +- CometBroadcastExchange (15) : : : : +- CometProject (14) : : : : +- CometFilter (13) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) : : : +- CometBroadcastExchange (21) : : : +- CometProject (20) : : : +- CometFilter (19) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : : +- CometNativeScan parquet spark_catalog.default.store (18) : : +- CometBroadcastExchange (27) : : +- CometProject (26) : : +- CometFilter (25) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : : +- CometNativeScan parquet spark_catalog.default.item (24) : +- CometBroadcastExchange (33) : +- CometProject (32) : +- CometFilter (31) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + : +- CometNativeScan parquet spark_catalog.default.promotion (30) :- CometHashAggregate (67) : +- CometExchange (66) : +- CometHashAggregate (65) @@ -65,12 +65,12 @@ : : : : : +- CometExchange (46) : : : : : +- CometProject (45) : : : : : +- CometFilter (44) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (43) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (43) : : : : +- ReusedExchange (50) : : : +- CometBroadcastExchange (56) : : : +- CometProject (55) : : : +- CometFilter (54) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (53) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (53) : : +- ReusedExchange (59) : +- ReusedExchange (62) +- CometHashAggregate (96) @@ -94,12 +94,12 @@ : : : : +- CometExchange (75) : : : : +- CometProject (74) : : : : +- CometFilter (73) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (72) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (72) : : : +- ReusedExchange (79) : : +- CometBroadcastExchange (85) : : +- CometProject (84) : : +- CometFilter (83) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (82) + : : +- CometNativeScan parquet spark_catalog.default.web_site (82) : +- ReusedExchange (88) +- ReusedExchange (91) @@ -124,7 +124,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -156,7 +156,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#1 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(12) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -184,7 +184,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(18) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#16, s_store_id#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -212,7 +212,7 @@ Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(24) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_current_price#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -240,7 +240,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(30) CometNativeScan parquet spark_catalog.default.promotion Output [2]: [p_promo_sk#21, p_channel_tv#22] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -302,7 +302,7 @@ Arguments: hashpartitioning(cs_item_sk#29, cs_order_number#31, 5), ENSURE_REQUIR Input [7]: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_order_number#31, cs_ext_sales_price#32, cs_net_profit#33, cs_sold_date_sk#34] Arguments: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_order_number#31, cs_ext_sales_price#32, cs_net_profit#33, cs_sold_date_sk#34], [cs_item_sk#29 ASC NULLS FIRST, cs_order_number#31 ASC NULLS FIRST] -(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(43) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#36, cr_order_number#37, cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -346,7 +346,7 @@ Arguments: [cs_sold_date_sk#34], [d_date_sk#41], Inner, BuildRight Input [9]: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_ext_sales_price#32, cs_net_profit#33, cs_sold_date_sk#34, cr_return_amount#38, cr_net_loss#39, d_date_sk#41] Arguments: [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_ext_sales_price#32, cs_net_profit#33, cr_return_amount#38, cr_net_loss#39], [cs_catalog_page_sk#28, cs_item_sk#29, cs_promo_sk#30, cs_ext_sales_price#32, cs_net_profit#33, cr_return_amount#38, cr_net_loss#39] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +(53) CometNativeScan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#42, cp_catalog_page_id#43] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -432,7 +432,7 @@ Arguments: hashpartitioning(ws_item_sk#52, ws_order_number#55, 5), ENSURE_REQUIR Input [7]: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_order_number#55, ws_ext_sales_price#56, ws_net_profit#57, ws_sold_date_sk#58] Arguments: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_order_number#55, ws_ext_sales_price#56, ws_net_profit#57, ws_sold_date_sk#58], [ws_item_sk#52 ASC NULLS FIRST, ws_order_number#55 ASC NULLS FIRST] -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(72) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#60, wr_order_number#61, wr_return_amt#62, wr_net_loss#63, wr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -476,7 +476,7 @@ Arguments: [ws_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight Input [9]: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_ext_sales_price#56, ws_net_profit#57, ws_sold_date_sk#58, wr_return_amt#62, wr_net_loss#63, d_date_sk#65] Arguments: [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_ext_sales_price#56, ws_net_profit#57, wr_return_amt#62, wr_net_loss#63], [ws_item_sk#52, ws_web_site_sk#53, ws_promo_sk#54, ws_ext_sales_price#56, ws_net_profit#57, wr_return_amt#62, wr_net_loss#63] -(82) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +(82) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#66, web_site_id#67] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] @@ -579,10 +579,10 @@ BroadcastExchange (108) +- * CometColumnarToRow (107) +- CometProject (106) +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (104) -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(104) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/extended.txt index ae6fc19b60..db569f1bf2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/extended.txt @@ -27,28 +27,28 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : +- CometNativeScan parquet spark_catalog.default.store : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -71,23 +71,23 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -110,22 +110,22 @@ CometColumnarToRow : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site + : : +- CometNativeScan parquet spark_catalog.default.web_site : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + +- CometNativeScan parquet spark_catalog.default.promotion Comet accelerated 123 out of 127 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt index 198646e0fd..2598a261c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt @@ -31,28 +31,28 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),cp_catalog_page_id] CometExchange [cp_catalog_page_id] #10 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] @@ -75,12 +75,12 @@ WholeStageCodegen (1) CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),web_site_id] @@ -105,11 +105,11 @@ WholeStageCodegen (1) CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt index 35974f7924..594f5acd53 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt @@ -20,11 +20,11 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : : +- CometBroadcastExchange (34) : : +- CometFilter (33) : : +- CometHashAggregate (32) @@ -44,11 +44,11 @@ : +- CometBroadcastExchange (40) : +- CometProject (39) : +- CometFilter (38) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : +- CometNativeScan parquet spark_catalog.default.customer (37) +- CometBroadcastExchange (46) +- CometProject (45) +- CometFilter (44) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns @@ -63,7 +63,7 @@ ReadSchema: struct Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -90,7 +90,7 @@ Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -281,18 +281,18 @@ BroadcastExchange (63) +- CometProject (61) +- CometBroadcastHashJoin (60) :- CometFilter (51) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) + : +- CometNativeScan parquet spark_catalog.default.date_dim (50) +- CometBroadcastExchange (59) +- CometProject (58) +- CometBroadcastHashJoin (57) - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) + :- CometNativeScan parquet spark_catalog.default.date_dim (52) +- CometBroadcastExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) + +- CometNativeScan parquet spark_catalog.default.date_dim (53) -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(50) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -303,13 +303,13 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(52) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#10, d_week_seq#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(53) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/extended.txt index 924b12cf01..4fabc2738e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/extended.txt @@ -19,32 +19,32 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin - : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : :- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin - : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : :- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometHashAggregate : +- CometExchange @@ -59,20 +59,20 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin - : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : :- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -87,19 +87,19 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + :- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 97 out of 101 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt index d016a5ee2d..06f1225067 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt @@ -23,24 +23,24 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cr_item_qty] #7 CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt index 5bafdc5586..63140cfc2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt @@ -14,27 +14,27 @@ : : : : +- CometBroadcastHashJoin (8) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (4) : : : +- CometBroadcastExchange (12) : : : +- CometFilter (11) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (10) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) + : +- CometNativeScan parquet spark_catalog.default.income_band (20) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) + +- CometNativeScan parquet spark_catalog.default.store_returns (27) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -49,7 +49,7 @@ Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2) Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#1, 16, true, false, true) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#5, 20, true, false, true) AS c_first_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#6, 30, true, false, true) AS c_last_name#9] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(4) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#10, ca_city#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -77,7 +77,7 @@ Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(10) CometNativeScan parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -101,7 +101,7 @@ Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -125,7 +125,7 @@ Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +(20) CometNativeScan parquet spark_catalog.default.income_band Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -157,7 +157,7 @@ Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_c Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(27) CometNativeScan parquet spark_catalog.default.store_returns Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/extended.txt index 5fce933878..b025c868d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/extended.txt @@ -13,23 +13,23 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : +- CometNativeScan parquet spark_catalog.default.income_band +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + +- CometNativeScan parquet spark_catalog.default.store_returns Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt index e43557c27d..9df82b0e98 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt @@ -15,21 +15,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt index c263f03d76..723a82c145 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt @@ -23,30 +23,30 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometProject (16) : : : : +- CometFilter (15) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (14) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + : +- CometNativeScan parquet spark_catalog.default.date_dim (32) +- CometBroadcastExchange (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) + +- CometNativeScan parquet spark_catalog.default.reason (38) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -65,7 +65,7 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(4) CometNativeScan parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -89,7 +89,7 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +(9) CometNativeScan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -113,7 +113,7 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(14) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -141,7 +141,7 @@ Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_s Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(20) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -169,7 +169,7 @@ Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status# Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(26) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -197,7 +197,7 @@ Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(32) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -225,7 +225,7 @@ Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +(38) CometNativeScan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#35, r_reason_desc#36] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -281,10 +281,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/extended.txt index e2383394d1..9d9ce62ec2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/extended.txt @@ -25,32 +25,32 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason + +- CometNativeScan parquet spark_catalog.default.reason Comet accelerated 50 out of 52 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt index dfcca32f03..071d1eba87 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt @@ -29,30 +29,30 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometBroadcastExchange [wp_web_page_sk] #4 CometFilter [wp_web_page_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #7 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt index 1af5e2612c..fb0df08038 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject (24) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometNativeScan parquet spark_catalog.default.item (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -65,7 +65,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -141,10 +141,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + +- CometNativeScan parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt index dfc810b108..88ef4a1db5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt @@ -19,14 +19,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 23 out of 28 eligible operators (82%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt index 29e2d72920..33e882bb0d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt @@ -25,12 +25,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt index fc6b12b834..3a671be2d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt @@ -19,11 +19,11 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) : +- BroadcastExchange (31) : +- * CometColumnarToRow (30) : +- CometHashAggregate (29) @@ -64,7 +64,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -92,7 +92,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -295,10 +295,10 @@ BroadcastExchange (56) +- * CometColumnarToRow (55) +- CometProject (54) +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) + +- CometNativeScan parquet spark_catalog.default.date_dim (52) -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(52) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/extended.txt index ddee139acf..37eea8ba60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/extended.txt @@ -20,15 +20,15 @@ HashAggregate : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- BroadcastExchange : +- CometColumnarToRow : +- CometHashAggregate @@ -44,11 +44,11 @@ HashAggregate : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : +- CometNativeScan parquet spark_catalog.default.customer +- BroadcastExchange +- CometColumnarToRow +- CometHashAggregate @@ -64,10 +64,10 @@ HashAggregate : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 55 out of 66 eligible operators (83%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt index f687139735..776f189141 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt @@ -26,15 +26,15 @@ WholeStageCodegen (4) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (1) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt index 2e49d20af4..25b4c305f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt @@ -18,19 +18,19 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) : : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) @@ -44,12 +44,12 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) : : : : : +- * CometColumnarToRow (65) @@ -64,12 +64,12 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) : : : : +- * CometColumnarToRow (86) @@ -84,12 +84,12 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) : : : +- * CometColumnarToRow (107) @@ -104,12 +104,12 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) : : +- * CometColumnarToRow (128) @@ -124,12 +124,12 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) +: : : +- CometNativeScan parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) : +- * CometColumnarToRow (149) @@ -144,12 +144,12 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) +: : +- CometNativeScan parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) +- * CometColumnarToRow (170) @@ -164,16 +164,16 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) + : +- CometNativeScan parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -188,7 +188,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -216,7 +216,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -244,7 +244,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -288,7 +288,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -315,7 +315,7 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -379,7 +379,7 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(47) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -406,7 +406,7 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(53) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -470,7 +470,7 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(68) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -497,7 +497,7 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(74) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -561,7 +561,7 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(89) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -588,7 +588,7 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(95) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -652,7 +652,7 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(110) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -679,7 +679,7 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(116) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -743,7 +743,7 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(131) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -770,7 +770,7 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(137) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -834,7 +834,7 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(152) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -861,7 +861,7 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(158) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/extended.txt index 1e1247665c..24f596de9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/extended.txt @@ -17,19 +17,19 @@ BroadcastNestedLoopJoin : : : : : : : : : +- CometBroadcastHashJoin : : : : : : : : : :- CometProject : : : : : : : : : : +- CometFilter -: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometProject : : : : : : : : : +- CometFilter -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometProject : : : : : : : : +- CometFilter -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : : +- BroadcastExchange : : : : : : +- CometColumnarToRow : : : : : : +- CometHashAggregate @@ -43,19 +43,19 @@ BroadcastNestedLoopJoin : : : : : : : : +- CometBroadcastHashJoin : : : : : : : : :- CometProject : : : : : : : : : +- CometFilter -: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometProject : : : : : : : : +- CometFilter -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : +- BroadcastExchange : : : : : +- CometColumnarToRow : : : : : +- CometHashAggregate @@ -69,19 +69,19 @@ BroadcastNestedLoopJoin : : : : : : : +- CometBroadcastHashJoin : : : : : : : :- CometProject : : : : : : : : +- CometFilter -: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometHashAggregate @@ -95,19 +95,19 @@ BroadcastNestedLoopJoin : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometProject : : : : : : : +- CometFilter -: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : : +- CometNativeScan parquet spark_catalog.default.store : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometHashAggregate @@ -121,19 +121,19 @@ BroadcastNestedLoopJoin : : : : : +- CometBroadcastHashJoin : : : : : :- CometProject : : : : : : +- CometFilter -: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : : +- CometNativeScan parquet spark_catalog.default.store : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometHashAggregate @@ -147,19 +147,19 @@ BroadcastNestedLoopJoin : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter -: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : : +- CometNativeScan parquet spark_catalog.default.time_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: : +- CometNativeScan parquet spark_catalog.default.store : +- BroadcastExchange : +- CometColumnarToRow : +- CometHashAggregate @@ -173,19 +173,19 @@ BroadcastNestedLoopJoin : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter -: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter -: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +: : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +: : +- CometNativeScan parquet spark_catalog.default.time_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter -: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +: +- CometNativeScan parquet spark_catalog.default.store +- BroadcastExchange +- CometColumnarToRow +- CometHashAggregate @@ -199,18 +199,18 @@ BroadcastNestedLoopJoin : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : +- CometNativeScan parquet spark_catalog.default.time_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 192 out of 206 eligible operators (93%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt index e3923fb3ea..cef8e0e760 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt @@ -19,19 +19,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -48,12 +48,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 @@ -71,12 +71,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 @@ -94,12 +94,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 @@ -117,12 +117,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 @@ -140,12 +140,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 @@ -163,12 +163,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt index 47ce0d4a91..58f1696153 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometNativeScan parquet spark_catalog.default.store (15) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + +- CometNativeScan parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt index 825b1ed81c..77b01e59ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -25,13 +25,13 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 27 out of 33 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt index 61bfd1d960..96c0c76bee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] CometFilter [i_item_sk,i_brand,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] @@ -31,11 +31,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt index b3f32555de..da194f2825 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt @@ -2,10 +2,10 @@ * Project (4) +- * CometColumnarToRow (3) +- CometFilter (2) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) + +- CometNativeScan parquet spark_catalog.default.reason (1) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +(1) CometNativeScan parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -33,10 +33,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + +- CometNativeScan parquet spark_catalog.default.store_sales (5) -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -84,10 +84,10 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + +- CometNativeScan parquet spark_catalog.default.store_sales (13) -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -135,10 +135,10 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) + +- CometNativeScan parquet spark_catalog.default.store_sales (21) -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(21) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -186,10 +186,10 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (29) + +- CometNativeScan parquet spark_catalog.default.store_sales (29) -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(29) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -237,10 +237,10 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (37) + +- CometNativeScan parquet spark_catalog.default.store_sales (37) -(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(37) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt index ed71033b26..53ba3252cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt @@ -7,7 +7,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -18,7 +18,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -29,7 +29,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -40,7 +40,7 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : :- ReusedSubquery : :- Subquery @@ -51,11 +51,11 @@ : : +- CometHashAggregate : : +- CometProject : : +- CometFilter -: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +: : +- CometNativeScan parquet spark_catalog.default.store_sales : :- ReusedSubquery : +- ReusedSubquery +- CometColumnarToRow +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason + +- CometNativeScan parquet spark_catalog.default.reason Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt index 558f5f4b36..9593d6ebd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 @@ -23,7 +23,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 @@ -49,7 +49,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 @@ -62,10 +62,10 @@ WholeStageCodegen (1) CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 CometColumnarToRow InputAdapter CometFilter [r_reason_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt index 08ae744b5a..73b54f439b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt @@ -13,19 +13,19 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) + : +- CometNativeScan parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometHashAggregate (43) @@ -39,16 +39,16 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) + : +- CometNativeScan parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -63,7 +63,7 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -91,7 +91,7 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -119,7 +119,7 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +(16) CometNativeScan parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -163,7 +163,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(26) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -190,7 +190,7 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/extended.txt index aa6c577ed7..977fcbbc6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/extended.txt @@ -12,19 +12,19 @@ Project : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : : +- CometNativeScan parquet spark_catalog.default.time_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page + : +- CometNativeScan parquet spark_catalog.default.web_page +- BroadcastExchange +- CometColumnarToRow +- CometHashAggregate @@ -38,18 +38,18 @@ Project : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : +- CometNativeScan parquet spark_catalog.default.time_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page + +- CometNativeScan parquet spark_catalog.default.web_page Comet accelerated 48 out of 51 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt index 0991e4e0dc..a13072cd91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt index cf532d2b41..30ecac0944 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt @@ -19,32 +19,32 @@ : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometProject (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometFilter (5) : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : : : +- CometNativeScan parquet spark_catalog.default.customer (15) : : +- CometBroadcastExchange (23) : : +- CometProject (22) : : +- CometFilter (21) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) + +- CometNativeScan parquet spark_catalog.default.household_demographics (32) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(1) CometNativeScan parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -84,7 +84,7 @@ Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -112,7 +112,7 @@ Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(15) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -136,7 +136,7 @@ Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -164,7 +164,7 @@ Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(26) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -192,7 +192,7 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(32) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#25, hd_buy_potential#26] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -252,10 +252,10 @@ BroadcastExchange (48) +- * CometColumnarToRow (47) +- CometProject (46) +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) + +- CometNativeScan parquet spark_catalog.default.date_dim (44) -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(44) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/extended.txt index a1c6b73d37..d08a306fbf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/extended.txt @@ -18,7 +18,7 @@ CometColumnarToRow : : : : : +- CometBroadcastHashJoin : : : : : :- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center : : : : : +- CometBroadcastExchange : : : : : +- CometFilter : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns @@ -27,25 +27,25 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : +- CometNativeScan parquet spark_catalog.default.customer_address : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : +- CometNativeScan parquet spark_catalog.default.customer_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + +- CometNativeScan parquet spark_catalog.default.household_demographics Comet accelerated 45 out of 47 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt index 3e9b8945da..0daeeb561b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt @@ -20,7 +20,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] @@ -31,23 +31,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #7 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk] #9 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt index cdc8dbc3bc..39a40b4db7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt @@ -14,7 +14,7 @@ : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometNativeScan parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -27,7 +27,7 @@ : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Condition : isnotnull(ws_item_sk#7) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + +- CometNativeScan parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/extended.txt index 20df9a8047..c3eeb08246 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/extended.txt @@ -15,11 +15,11 @@ CometColumnarToRow : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -33,10 +33,10 @@ CometColumnarToRow : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 35 out of 38 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt index 4ec5755df7..c411db0635 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] @@ -37,5 +37,5 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt index 799d860c95..3f38adcd39 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt @@ -11,19 +11,19 @@ : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : +- CometNativeScan parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) + +- CometNativeScan parquet spark_catalog.default.reason (12) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -73,7 +73,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +(12) CometNativeScan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/extended.txt index 335f2765d7..7203a678b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/extended.txt @@ -10,15 +10,15 @@ CometColumnarToRow : :- CometSort : : +- CometExchange : : +- CometProject - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : +- CometNativeScan parquet spark_catalog.default.store_sales : +- CometSort : +- CometExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : +- CometNativeScan parquet spark_catalog.default.store_returns +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason + +- CometNativeScan parquet spark_catalog.default.reason Comet accelerated 21 out of 21 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt index d1de4f3475..c0b74010df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt @@ -12,13 +12,13 @@ WholeStageCodegen (1) CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometExchange [ss_item_sk,ss_ticket_number] #2 CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometExchange [sr_item_sk,sr_ticket_number] #3 CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [r_reason_sk] #4 CometProject [r_reason_sk] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt index 8164e345a6..1caeb8ca76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) + +- CometNativeScan parquet spark_catalog.default.web_site (29) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -92,7 +92,7 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(12) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -115,7 +115,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +(29) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt index eac4939621..e5c7b8688a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt @@ -18,26 +18,26 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometSort : : : +- CometExchange : : : +- CometProject - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site + +- CometNativeScan parquet spark_catalog.default.web_site Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt index 629178d106..feab73bcd6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [wr_order_number] CometExchange [wr_order_number] #4 CometProject [wr_order_number] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt index 24e37a8483..39f45f7a74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt @@ -18,14 +18,14 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : : +- CometProject (14) : : : : +- CometSortMergeJoin (13) : : : : :- CometSort (10) : : : : : +- CometExchange (9) : : : : : +- CometProject (8) : : : : : +- CometFilter (7) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) : : : : +- CometSort (12) : : : : +- ReusedExchange (11) : : : +- CometProject (28) @@ -34,7 +34,7 @@ : : : : +- CometExchange (19) : : : : +- CometProject (18) : : : : +- CometFilter (17) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (16) : : : +- CometProject (26) : : : +- CometSortMergeJoin (25) : : : :- CometSort (22) @@ -44,18 +44,18 @@ : : +- CometBroadcastExchange (33) : : +- CometProject (32) : : +- CometFilter (31) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) : +- CometBroadcastExchange (39) : +- CometProject (38) : +- CometFilter (37) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) + : +- CometNativeScan parquet spark_catalog.default.customer_address (36) +- CometBroadcastExchange (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) + +- CometNativeScan parquet spark_catalog.default.web_site (42) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -78,7 +78,7 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -122,7 +122,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +(16) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -182,7 +182,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -210,7 +210,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(36) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#21, ca_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -238,7 +238,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +(42) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#23, web_company_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt index 6ff8eba58f..cbba03d28d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt @@ -17,49 +17,49 @@ CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : : +- CometProject : : : : +- CometSortMergeJoin : : : : :- CometSort : : : : : +- CometExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : : +- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometProject : : : +- CometSortMergeJoin : : : :- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns : : : +- CometProject : : : +- CometSortMergeJoin : : : :- CometSort : : : : +- CometExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : : +- CometSort : : : +- CometExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales + : : : +- CometNativeScan parquet spark_catalog.default.web_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site + +- CometNativeScan parquet spark_catalog.default.web_site Comet accelerated 59 out of 61 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt index 6c1b1bccfa..15f29c507c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt @@ -21,14 +21,14 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 CometProject [wr_order_number] @@ -37,7 +37,7 @@ WholeStageCodegen (2) CometExchange [wr_order_number] #4 CometProject [wr_order_number] CometFilter [wr_order_number,wr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] @@ -47,12 +47,12 @@ WholeStageCodegen (2) CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt index f90d8b12b9..db701aae34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt @@ -11,22 +11,22 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + : +- CometNativeScan parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) + +- CometNativeScan parquet spark_catalog.default.store (16) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -69,7 +69,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -97,7 +97,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/extended.txt index 0f623c9021..170d3b7879 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/extended.txt @@ -10,18 +10,18 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales + : : : +- CometNativeScan parquet spark_catalog.default.store_sales : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : +- CometNativeScan parquet spark_catalog.default.household_demographics : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim + : +- CometNativeScan parquet spark_catalog.default.time_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + +- CometNativeScan parquet spark_catalog.default.store Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt index bbbd07245e..b40f41659f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt index 420f37fccc..5e84af4892 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt @@ -15,7 +15,7 @@ : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) + : +- CometNativeScan parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) @@ -33,7 +33,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] ReadSchema: struct -(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(2) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -149,10 +149,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + +- CometNativeScan parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/extended.txt index e1aefb8382..355bb7fda2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/extended.txt @@ -16,11 +16,11 @@ CometColumnarToRow : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometSort +- CometHashAggregate +- CometExchange @@ -32,6 +32,6 @@ CometColumnarToRow +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 30 out of 33 eligible operators (90%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt index a7de047b25..3088193cfb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt @@ -20,11 +20,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] CometExchange [cs_bill_customer_sk,cs_item_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt index 4cc725ef2b..1eacde21e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt @@ -20,11 +20,11 @@ : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -67,7 +67,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -150,10 +150,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) + +- CometNativeScan parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(27) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt index 4c972848e7..07eca8ca5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt @@ -21,14 +21,14 @@ CometColumnarToRow : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 25 out of 29 eligible operators (86%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt index af05a7d8c6..94361b16ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt @@ -29,12 +29,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt index 7e57135c10..3447b6efa3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt @@ -13,24 +13,24 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) : +- CometBroadcastExchange (16) : +- CometFilter (15) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- CometNativeScan parquet spark_catalog.default.call_center (14) +- CometBroadcastExchange (22) +- CometProject (21) +- CometFilter (20) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -41,7 +41,7 @@ ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customer_sk#14], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(24) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_county#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(30) CometNativeScan parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/extended.txt index 2cdc75e15e..90b468da99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/extended.txt @@ -11,7 +11,7 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -21,11 +21,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometUnion : : :- CometProject @@ -35,7 +35,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -43,14 +43,14 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt index 38f41d10a5..031c06de9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -25,11 +25,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [customer_sk] #5 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] @@ -45,8 +45,8 @@ WholeStageCodegen (1) CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt index 49c1c8e0d7..47b7928d4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,7 +51,7 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#23, year_total#24] Right output [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -325,7 +325,7 @@ Arguments: [customer_id#23], [customer_id#68], Inner, BuildRight Input [9]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, customer_id#68, year_total#69] Arguments: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69], [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/extended.txt index 476c7be954..18c1c90d20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -23,10 +23,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -37,7 +37,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -45,10 +45,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -60,14 +60,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -78,13 +78,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 79 out of 85 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt index 157d1d587e..26364c9ff3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -27,10 +27,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] @@ -51,10 +51,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -66,7 +66,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt index 410aecff1b..e1a264a366 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject (23) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt index 3f41c97ff5..282edf0ce0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt index 58ac81ba0d..36cef3bca1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt index ea550c6c74..5b68388c2a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt @@ -17,7 +17,7 @@ : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -32,7 +32,7 @@ : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) @@ -42,11 +42,11 @@ : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -60,12 +60,12 @@ : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -83,7 +83,7 @@ +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -121,7 +121,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(72) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(103) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(108) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) + +- CometNativeScan parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(112) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#103] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) + +- CometNativeScan parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(117) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) + +- CometNativeScan parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(122) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/extended.txt index f56d229b68..5011719f37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -23,7 +23,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -31,7 +31,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -51,13 +51,13 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -74,11 +74,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -89,15 +89,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -108,20 +108,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -138,11 +138,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -153,15 +153,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -172,16 +172,16 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter : +- ReusedSubquery @@ -204,13 +204,13 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -227,11 +227,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -242,15 +242,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -261,20 +261,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -291,11 +291,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -306,15 +306,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -325,15 +325,15 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter : +- ReusedSubquery - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 298 out of 327 eligible operators (91%). Final plan contains 10 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt index fb9abae378..4ae30a0feb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -75,11 +75,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -90,11 +90,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -109,13 +109,13 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt index a727c5e743..324afaadcf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt @@ -24,7 +24,7 @@ : : : : +- CometProject (45) : : : : +- CometBroadcastHashJoin (44) : : : : :- CometFilter (4) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) @@ -39,7 +39,7 @@ : : : : : : +- CometBroadcastExchange (24) : : : : : : +- CometBroadcastHashJoin (23) : : : : : : :- CometFilter (8) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : : +- CometBroadcastExchange (22) : : : : : : +- CometProject (21) : : : : : : +- CometBroadcastHashJoin (20) @@ -49,11 +49,11 @@ : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : : +- CometBroadcastExchange (13) : : : : : : : +- CometFilter (12) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : : +- CometBroadcastExchange (19) : : : : : : +- CometProject (18) : : : : : : +- CometFilter (17) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : : +- ReusedExchange (27) : : : : +- CometBroadcastExchange (41) : : : : +- CometProject (40) @@ -67,12 +67,12 @@ : : : +- CometBroadcastExchange (52) : : : +- CometBroadcastHashJoin (51) : : : :- CometFilter (49) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : : +- CometNativeScan parquet spark_catalog.default.item (48) : : : +- ReusedExchange (50) : : +- CometBroadcastExchange (58) : : +- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (55) : :- CometFilter (78) : : +- CometHashAggregate (77) : : +- CometExchange (76) @@ -135,7 +135,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -158,7 +158,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -181,7 +181,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -205,7 +205,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -348,7 +348,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -380,7 +380,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -703,7 +703,7 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer : +- CometBroadcastExchange (131) : +- CometProject (130) : +- CometFilter (129) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (128) + : +- CometNativeScan parquet spark_catalog.default.date_dim (128) +- CometProject (137) +- CometBroadcastHashJoin (136) :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (134) @@ -736,7 +736,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#152), dynamicpruningexpression(cs_sold_date_sk#152 IN dynamicpruning#153)] ReadSchema: struct -(128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(128) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#154, d_year#155] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -812,10 +812,10 @@ BroadcastExchange (147) +- * CometColumnarToRow (146) +- CometProject (145) +- CometFilter (144) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) + +- CometNativeScan parquet spark_catalog.default.date_dim (143) -(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(143) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#154, d_year#155] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -844,10 +844,10 @@ BroadcastExchange (152) +- * CometColumnarToRow (151) +- CometProject (150) +- CometFilter (149) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (148) + +- CometNativeScan parquet spark_catalog.default.date_dim (148) -(148) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(148) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -874,10 +874,10 @@ BroadcastExchange (157) +- * CometColumnarToRow (156) +- CometProject (155) +- CometFilter (154) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (153) + +- CometNativeScan parquet spark_catalog.default.date_dim (153) -(153) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(153) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#168] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/extended.txt index 83deb45432..6c5a02babc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/extended.txt @@ -22,7 +22,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -31,11 +31,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -43,7 +43,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -59,12 +59,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -81,11 +81,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -96,15 +96,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -115,20 +115,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -145,11 +145,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -160,15 +160,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -179,15 +179,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -205,7 +205,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -222,11 +222,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -237,15 +237,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -256,20 +256,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -286,11 +286,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -301,15 +301,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -320,15 +320,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -346,7 +346,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -363,11 +363,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -378,15 +378,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -397,20 +397,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -427,11 +427,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -442,15 +442,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -461,15 +461,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -491,7 +491,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -500,11 +500,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -512,7 +512,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -528,12 +528,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -550,11 +550,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -565,15 +565,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -584,20 +584,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -614,11 +614,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -629,15 +629,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -648,15 +648,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -674,7 +674,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -691,11 +691,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -706,15 +706,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -725,20 +725,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -755,11 +755,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -770,15 +770,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -789,15 +789,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -815,7 +815,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -832,11 +832,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -847,15 +847,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -866,20 +866,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -896,11 +896,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -911,15 +911,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -930,15 +930,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -960,7 +960,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -969,11 +969,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -981,7 +981,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -997,12 +997,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1019,11 +1019,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1034,15 +1034,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1053,20 +1053,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1083,11 +1083,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1098,15 +1098,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1117,15 +1117,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -1143,7 +1143,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1160,11 +1160,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1175,15 +1175,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1194,20 +1194,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1224,11 +1224,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1239,15 +1239,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1258,15 +1258,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -1284,7 +1284,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1301,11 +1301,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1316,15 +1316,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1335,20 +1335,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -1365,11 +1365,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1380,15 +1380,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -1399,15 +1399,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -1429,7 +1429,7 @@ CometColumnarToRow : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : :- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -1438,11 +1438,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -1450,7 +1450,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometHashAggregate : : +- CometExchange : : +- CometHashAggregate @@ -1466,12 +1466,12 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1488,11 +1488,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1503,15 +1503,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1522,20 +1522,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1552,11 +1552,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1567,15 +1567,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1586,15 +1586,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : :- CometFilter : : : +- ReusedSubquery : : +- CometHashAggregate @@ -1612,7 +1612,7 @@ CometColumnarToRow : : : : +- CometProject : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometHashAggregate @@ -1629,11 +1629,11 @@ CometColumnarToRow : : : : : : : +- CometColumnarToRow : : : : : : : +- CometProject : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : +- CometBroadcastExchange : : : : : : +- CometBroadcastHashJoin : : : : : : :- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometBroadcastHashJoin @@ -1644,15 +1644,15 @@ CometColumnarToRow : : : : : : : : +- ReusedSubquery : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1663,20 +1663,20 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1693,11 +1693,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1708,15 +1708,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1727,15 +1727,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -1753,7 +1753,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1770,11 +1770,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1785,15 +1785,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1804,20 +1804,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -1834,11 +1834,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -1849,15 +1849,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -1868,15 +1868,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -1898,7 +1898,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : :- CometProject : : : +- CometBroadcastHashJoin : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -1907,11 +1907,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales @@ -1919,7 +1919,7 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -1935,12 +1935,12 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -1957,11 +1957,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -1972,15 +1972,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -1991,20 +1991,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -2021,11 +2021,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -2036,15 +2036,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -2055,15 +2055,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- CometFilter : : +- ReusedSubquery : +- CometHashAggregate @@ -2081,7 +2081,7 @@ CometColumnarToRow : : : +- CometProject : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometHashAggregate @@ -2098,11 +2098,11 @@ CometColumnarToRow : : : : : : +- CometColumnarToRow : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : +- CometBroadcastExchange : : : : : +- CometBroadcastHashJoin : : : : : :- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometBroadcastHashJoin @@ -2113,15 +2113,15 @@ CometColumnarToRow : : : : : : : +- ReusedSubquery : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -2132,20 +2132,20 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -2162,11 +2162,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -2177,15 +2177,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -2196,15 +2196,15 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometFilter : +- ReusedSubquery +- CometHashAggregate @@ -2222,7 +2222,7 @@ CometColumnarToRow : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometBroadcastHashJoin : : :- CometHashAggregate @@ -2239,11 +2239,11 @@ CometColumnarToRow : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -2254,15 +2254,15 @@ CometColumnarToRow : : : : : : +- ReusedSubquery : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometBroadcastHashJoin @@ -2273,20 +2273,20 @@ CometColumnarToRow : : : : +- ReusedSubquery : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin : :- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometBroadcastHashJoin : :- CometHashAggregate @@ -2303,11 +2303,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -2318,15 +2318,15 @@ CometColumnarToRow : : : : : +- ReusedSubquery : : : : +- CometBroadcastExchange : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometBroadcastHashJoin @@ -2337,14 +2337,14 @@ CometColumnarToRow : : : +- ReusedSubquery : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : +- CometNativeScan parquet spark_catalog.default.item : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 2127 out of 2302 eligible operators (92%). Final plan contains 46 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt index 758e917335..8f7b98b200 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt @@ -34,11 +34,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] @@ -61,12 +61,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #5 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #6 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -85,11 +85,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -100,11 +100,11 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 CometProject [i_brand_id,i_class_id,i_category_id] @@ -119,12 +119,12 @@ WholeStageCodegen (1) CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #5 CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt index 72e9bd1b4a..5810beb47e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt @@ -22,26 +22,26 @@ : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + : +- CometNativeScan parquet spark_catalog.default.item (32) :- CometHashAggregate (66) : +- CometExchange (65) : +- CometHashAggregate (64) @@ -65,7 +65,7 @@ : : : +- CometBroadcastExchange (55) : : : +- CometProject (54) : : : +- CometFilter (53) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (52) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (52) : : +- ReusedExchange (58) : +- ReusedExchange (61) :- CometHashAggregate (92) @@ -91,7 +91,7 @@ : : : +- CometBroadcastExchange (81) : : : +- CometProject (80) : : : +- CometFilter (79) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (78) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (78) : : +- ReusedExchange (84) : +- ReusedExchange (87) :- CometHashAggregate (118) @@ -117,7 +117,7 @@ : : : +- CometBroadcastExchange (107) : : : +- CometProject (106) : : : +- CometFilter (105) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (104) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (104) : : +- ReusedExchange (110) : +- ReusedExchange (113) +- CometHashAggregate (143) @@ -144,7 +144,7 @@ : +- ReusedExchange (133) +- CometBroadcastExchange (138) +- CometFilter (137) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (136) + +- CometNativeScan parquet spark_catalog.default.item (136) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -159,7 +159,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(24) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt index cd52b2cd12..754a4fad60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt @@ -18,14 +18,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 22 out of 27 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt index cf18e68a3d..6fadc67351 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt index f85c898208..161f471629 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt @@ -17,14 +17,14 @@ TakeOrderedAndProject (26) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometNativeScan parquet spark_catalog.default.item (9) +- BroadcastExchange (18) +- * CometColumnarToRow (17) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) + +- CometNativeScan parquet spark_catalog.default.warehouse (16) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -67,7 +67,7 @@ Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_pro (15) CometColumnarToRow [codegen id : 2] Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(16) CometNativeScan parquet spark_catalog.default.warehouse Output: [] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) + +- CometNativeScan parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(27) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/extended.txt index 6cb89465d2..dfdad622b8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/extended.txt @@ -18,17 +18,17 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- BroadcastExchange +- CometColumnarToRow - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + +- CometNativeScan parquet spark_catalog.default.warehouse Comet accelerated 19 out of 28 eligible operators (67%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt index 221c6063ce..16246ec037 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt @@ -24,18 +24,18 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + CometNativeScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt index bf16cc4d21..69e1feda78 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt @@ -18,14 +18,14 @@ : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : : +- CometNativeScan parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) + : +- CometNativeScan parquet spark_catalog.default.warehouse (15) :- CometHashAggregate (29) : +- CometExchange (28) : +- CometHashAggregate (27) @@ -60,7 +60,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + +- CometNativeScan parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(34) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/extended.txt index 3d9d23d2cc..20468961d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/extended.txt @@ -20,22 +20,22 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : +- CometNativeScan parquet spark_catalog.default.household_demographics +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + +- CometNativeScan parquet spark_catalog.default.customer Comet accelerated 35 out of 37 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt index d57afd4fcf..a12adcdfec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt index b43ed9b192..3583ae80be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt @@ -15,7 +15,7 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) @@ -23,7 +23,7 @@ TakeOrderedAndProject (47) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) @@ -40,15 +40,15 @@ TakeOrderedAndProject (47) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -66,7 +66,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(29) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#22] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(36) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) + +- CometNativeScan parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(48) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/extended.txt index a6f33d6f7e..ee476d727a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/extended.txt @@ -14,7 +14,7 @@ TakeOrderedAndProject : : : :- CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometBroadcastHashJoin @@ -24,11 +24,11 @@ TakeOrderedAndProject : : : : : +- CometColumnarToRow : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometProject @@ -38,7 +38,7 @@ TakeOrderedAndProject : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometProject @@ -48,16 +48,16 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- CometColumnarToRow : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- BroadcastExchange +- CometColumnarToRow +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt index fea6a32741..24572a88a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt index e22a5c492c..19ef78cfce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt @@ -12,7 +12,7 @@ : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) @@ -20,7 +20,7 @@ : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) @@ -34,14 +34,14 @@ : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) +- CometBroadcastExchange (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,7 +59,7 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] ReadSchema: struct -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customsk#14], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(24) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20, ca_state#22] Arguments: [c_current_cdemo_sk#2, ca_state#22], [c_current_cdemo_sk#2, ca_state#22] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(30) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/extended.txt index 2cdc75e15e..90b468da99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/extended.txt @@ -11,7 +11,7 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometBroadcastHashJoin : : : :- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometBroadcastHashJoin @@ -21,11 +21,11 @@ CometColumnarToRow : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometUnion : : :- CometProject @@ -35,7 +35,7 @@ CometColumnarToRow : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometProject : : +- CometBroadcastHashJoin : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -43,14 +43,14 @@ CometColumnarToRow : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : +- CometNativeScan parquet spark_catalog.default.customer_address +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + +- CometNativeScan parquet spark_catalog.default.customer_demographics Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt index 69e8d4868c..33cc09317c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -25,11 +25,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [customsk] #5 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] @@ -45,8 +45,8 @@ WholeStageCodegen (1) CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt index 2562693a01..5b7d0a4b01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt @@ -23,15 +23,15 @@ TakeOrderedAndProject (43) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : : +- CometNativeScan parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- CometNativeScan parquet spark_catalog.default.store (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -56,7 +56,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -418,10 +418,10 @@ BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometProject (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt index f7d3371108..c098b443fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt @@ -36,11 +36,11 @@ TakeOrderedAndProject : : : : +- CometColumnarToRow : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -59,11 +59,11 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometSort : +- CometColumnarExchange : +- HashAggregate @@ -88,7 +88,7 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -106,7 +106,7 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -141,11 +141,11 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -164,11 +164,11 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometSort +- CometColumnarExchange +- HashAggregate @@ -193,7 +193,7 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -211,6 +211,6 @@ TakeOrderedAndProject +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 138 out of 196 eligible operators (70%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt index b3013059b0..7e155be3c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt @@ -48,11 +48,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter BroadcastExchange #8 WholeStageCodegen (3) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt index d1daee98c1..7285a59f3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +(14) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) + +- CometNativeScan parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(49) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt index de7d26cd06..3ab8670a2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -30,13 +30,13 @@ TakeOrderedAndProject : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : : +- CometNativeScan parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -54,7 +54,7 @@ TakeOrderedAndProject : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : +- CometNativeScan parquet spark_catalog.default.item : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -62,13 +62,13 @@ TakeOrderedAndProject : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + : +- CometNativeScan parquet spark_catalog.default.call_center +- BroadcastExchange +- Project +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] @@ -86,7 +86,7 @@ TakeOrderedAndProject : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : +- CometNativeScan parquet spark_catalog.default.item : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -94,12 +94,12 @@ TakeOrderedAndProject : : +- BroadcastExchange : : +- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center + +- CometNativeScan parquet spark_catalog.default.call_center Comet accelerated 75 out of 97 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt index d655789fe6..1c40ce03de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] @@ -38,13 +38,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt index 33c3a5de2e..eb318f9fd4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt @@ -26,11 +26,11 @@ : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : : +- CometNativeScan parquet spark_catalog.default.store (14) : :- CometHashAggregate (41) : : +- CometExchange (40) : : +- CometHashAggregate (39) @@ -49,7 +49,7 @@ : : +- CometBroadcastExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (33) + : : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) : +- CometHashAggregate (64) : +- CometExchange (63) : +- CometHashAggregate (62) @@ -67,12 +67,12 @@ : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (45) : : : +- CometProject (49) : : : +- CometFilter (48) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) : : +- ReusedExchange (53) : +- CometBroadcastExchange (59) : +- CometProject (58) : +- CometFilter (57) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (56) + : +- CometNativeScan parquet spark_catalog.default.web_site (56) :- CometHashAggregate (73) : +- CometExchange (72) : +- CometHashAggregate (71) @@ -121,7 +121,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -149,7 +149,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -239,7 +239,7 @@ Arguments: [date_sk#37], [d_date_sk#52], Inner, BuildRight Input [7]: [page_sk#36, date_sk#37, sales_price#38, profit#39, return_amt#40, net_loss#41, d_date_sk#52] Arguments: [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41], [page_sk#36, sales_price#38, profit#39, return_amt#40, net_loss#41] -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +(33) CometNativeScan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -308,7 +308,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(22) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_current_price#17, i_category#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) + +- CometNativeScan parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(46) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/extended.txt index f5b69fc6cf..5cce29dd89 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/extended.txt @@ -14,10 +14,10 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -33,18 +33,18 @@ CometColumnarToRow : : : +- CometHashAggregate : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter : : +- ReusedSubquery - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometBroadcastHashJoin :- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometFilter +- CometHashAggregate @@ -52,6 +52,6 @@ CometColumnarToRow +- CometHashAggregate +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 48 out of 52 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt index 2978e30c1c..04824dfed4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt @@ -16,10 +16,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt index 077daeca3a..e9a14a21a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt @@ -49,7 +49,7 @@ : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,51 +62,51 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (31) : : : : : : : : : : : : : : +- CometBroadcastExchange (39) : : : : : : : : : : : : : : +- CometProject (38) : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (36) : : : : : : : : : : : : : +- CometBroadcastExchange (44) : : : : : : : : : : : : : +- CometFilter (43) - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (42) : : : : : : : : : : : : +- CometBroadcastExchange (49) : : : : : : : : : : : : +- CometFilter (48) - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (47) : : : : : : : : : : : +- ReusedExchange (52) : : : : : : : : : : +- CometBroadcastExchange (58) : : : : : : : : : : +- CometProject (57) : : : : : : : : : : +- CometFilter (56) - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (55) : : : : : : : : : +- ReusedExchange (61) : : : : : : : : +- CometBroadcastExchange (66) : : : : : : : : +- CometFilter (65) - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (64) : : : : : : : +- CometBroadcastExchange (71) : : : : : : : +- CometFilter (70) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (69) : : : : : : +- ReusedExchange (74) : : : : : +- CometBroadcastExchange (80) : : : : : +- CometProject (79) : : : : : +- CometFilter (78) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (77) : : : : +- ReusedExchange (83) : : : +- CometBroadcastExchange (88) : : : +- CometFilter (87) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (86) : : +- ReusedExchange (91) : +- CometBroadcastExchange (97) : +- CometProject (96) : +- CometFilter (95) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) + : +- CometNativeScan parquet spark_catalog.default.item (94) +- CometSort (171) +- CometExchange (170) +- CometHashAggregate (169) @@ -152,7 +152,7 @@ : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (107) : : : : : : : : : : : : : : : +- CometSort (118) : : : : : : : : : : : : : : : +- CometProject (117) : : : : : : : : : : : : : : : +- CometFilter (116) @@ -160,7 +160,7 @@ : : : : : : : : : : : : : : : +- ReusedExchange (114) : : : : : : : : : : : : : : +- CometBroadcastExchange (123) : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (121) : : : : : : : : : : : : : +- ReusedExchange (126) : : : : : : : : : : : : +- ReusedExchange (129) : : : : : : : : : : : +- ReusedExchange (132) @@ -193,7 +193,7 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -225,7 +225,7 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +(11) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -248,7 +248,7 @@ Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] -(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -315,7 +315,7 @@ Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -339,7 +339,7 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(36) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -367,7 +367,7 @@ Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(42) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -391,7 +391,7 @@ Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(47) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -427,7 +427,7 @@ Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(55) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#48, cd_marital_status#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -467,7 +467,7 @@ Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_stat Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(64) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#53] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -491,7 +491,7 @@ Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(69) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -527,7 +527,7 @@ Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +(77) CometNativeScan parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -567,7 +567,7 @@ Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +(86) CometNativeScan parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#70] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -603,7 +603,7 @@ Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(94) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -665,7 +665,7 @@ Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +(107) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -726,7 +726,7 @@ Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(121) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -962,10 +962,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dy BroadcastExchange (180) +- * CometColumnarToRow (179) +- CometFilter (178) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) + +- CometNativeScan parquet spark_catalog.default.date_dim (177) -(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(177) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -987,10 +987,10 @@ Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN BroadcastExchange (184) +- * CometColumnarToRow (183) +- CometFilter (182) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) + +- CometNativeScan parquet spark_catalog.default.date_dim (181) -(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(181) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/extended.txt index 059acab385..af08477984 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/extended.txt @@ -50,10 +50,10 @@ CometColumnarToRow : : : : : : : : : : : : : : : : : : +- BroadcastExchange : : : : : : : : : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter @@ -66,63 +66,63 @@ CometColumnarToRow : : : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : +- CometProject : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometProject : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : : : +- CometNativeScan parquet spark_catalog.default.income_band : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : : +- CometNativeScan parquet spark_catalog.default.income_band : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometSort +- CometExchange +- CometHashAggregate @@ -170,10 +170,10 @@ CometColumnarToRow : : : : : : : : : : : : : : : : : +- BroadcastExchange : : : : : : : : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : +- CometFilter @@ -186,62 +186,62 @@ CometColumnarToRow : : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales : : : : : : : : : : : : : : : +- CometSort : : : : : : : : : : : : : : : +- CometExchange : : : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns : : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : : +- CometProject : : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store : : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : : +- CometFilter - : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometProject : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometProject : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion : : : : : : +- CometBroadcastExchange : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : : +- CometBroadcastExchange : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address + : : : +- CometNativeScan parquet spark_catalog.default.customer_address : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : : +- CometNativeScan parquet spark_catalog.default.income_band : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band + : +- CometNativeScan parquet spark_catalog.default.income_band +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 238 out of 242 eligible operators (98%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt index 46fe063430..87b73a92e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt @@ -54,10 +54,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -70,51 +70,51 @@ WholeStageCodegen (1) CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #12 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #12 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #13 CometBroadcastExchange [p_promo_sk] #14 CometFilter [p_promo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometBroadcastExchange [ib_income_band_sk] #17 CometFilter [ib_income_band_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] ReusedExchange [ib_income_band_sk] #17 CometBroadcastExchange [i_item_sk,i_product_name] #18 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] @@ -164,10 +164,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -175,7 +175,7 @@ WholeStageCodegen (1) ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 CometBroadcastExchange [d_date_sk,d_year] #23 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #10 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 ReusedExchange [d_date_sk,d_year] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt index 46f670a0bd..cea711acbd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt @@ -24,15 +24,15 @@ TakeOrderedAndProject (74) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometNativeScan parquet spark_catalog.default.item (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -87,7 +87,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -110,7 +110,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -136,7 +136,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -378,10 +378,10 @@ BroadcastExchange (71) +- * CometColumnarToRow (70) +- CometProject (69) +- CometFilter (68) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (67) + +- CometNativeScan parquet spark_catalog.default.date_dim (67) -(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(67) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt index 191d0ef18a..2b090d47d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt @@ -24,17 +24,17 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- BroadcastHashJoin : :- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- BroadcastExchange : +- Project : +- Filter @@ -55,11 +55,11 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim :- HashAggregate : +- CometColumnarToRow : +- CometColumnarExchange @@ -80,17 +80,17 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project : +- BroadcastHashJoin : :- CometColumnarToRow : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- BroadcastExchange : +- Project : +- Filter @@ -111,11 +111,11 @@ TakeOrderedAndProject : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : : +- CometNativeScan parquet spark_catalog.default.store : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- HashAggregate +- CometColumnarToRow +- CometColumnarExchange @@ -136,17 +136,17 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project +- BroadcastHashJoin :- CometColumnarToRow : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- BroadcastExchange +- Project +- Filter @@ -167,10 +167,10 @@ TakeOrderedAndProject : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store + : +- CometNativeScan parquet spark_catalog.default.store +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 102 out of 156 eligible operators (65%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt index 2832c5a348..73311c31f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt @@ -36,11 +36,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) @@ -49,7 +49,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (2) @@ -75,7 +75,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastExchange [s_store_sk,s_state] #9 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #5 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt index ee3242f2b1..41ef5269e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt @@ -30,39 +30,39 @@ : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) : : : : : : : +- CometBroadcastExchange (15) : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) : : : : : : +- CometBroadcastExchange (21) : : : : : : +- CometProject (20) : : : : : : +- CometFilter (19) - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) : : : : : +- CometBroadcastExchange (27) : : : : : +- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) : : : : +- CometBroadcastExchange (33) : : : : +- CometProject (32) : : : : +- CometFilter (31) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) : : : +- CometBroadcastExchange (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) : : +- CometBroadcastExchange (43) : : +- CometFilter (42) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) : +- CometBroadcastExchange (48) : +- CometFilter (47) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) +- CometSort (57) +- CometExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales @@ -77,10 +77,10 @@ ReadSchema: struct @@ -102,7 +102,7 @@ Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < c Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -126,7 +126,7 @@ Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#16, i_item_desc#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -150,7 +150,7 @@ Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#18, cd_marital_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -178,7 +178,7 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#20, hd_buy_potential#21] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -206,7 +206,7 @@ Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -234,7 +234,7 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -258,7 +258,7 @@ Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#28, d_date#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -282,7 +282,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) + +- CometNativeScan parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(65) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/extended.txt index 9000d27d1b..4f2941b209 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/extended.txt @@ -32,41 +32,41 @@ CometColumnarToRow : : : : : : : : : : +- CometColumnarToRow : : : : : : : : : : +- CometProject : : : : : : : : : : +- CometFilter - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : : : : : : +- CometBroadcastExchange : : : : : : : : : +- CometFilter - : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory : : : : : : : : +- CometBroadcastExchange : : : : : : : : +- CometFilter - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse : : : : : : : +- CometBroadcastExchange : : : : : : : +- CometFilter - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item : : : : : : +- CometBroadcastExchange : : : : : : +- CometProject : : : : : : +- CometFilter - : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics : : : : : +- CometBroadcastExchange : : : : : +- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics : : : : +- CometBroadcastExchange : : : : +- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion + : +- CometNativeScan parquet spark_catalog.default.promotion +- CometSort +- CometExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns + +- CometNativeScan parquet spark_catalog.default.catalog_returns Comet accelerated 66 out of 68 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt index 678f8c67df..e7af25252a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt index b0ae8bb695..280f5e12fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,7 +51,7 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(18) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(36) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +(54) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) + +- CometNativeScan parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(77) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/extended.txt index 476c7be954..18c1c90d20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/extended.txt @@ -15,7 +15,7 @@ CometColumnarToRow : : : : +- CometBroadcastHashJoin : : : : :- CometProject : : : : : +- CometFilter - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : : +- CometBroadcastExchange : : : : +- CometFilter : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -23,10 +23,10 @@ CometColumnarToRow : : : : +- BroadcastExchange : : : : +- CometColumnarToRow : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- CometBroadcastExchange : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometHashAggregate : : +- CometExchange @@ -37,7 +37,7 @@ CometColumnarToRow : : : +- CometBroadcastHashJoin : : : :- CometProject : : : : +- CometFilter - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : : +- CometNativeScan parquet spark_catalog.default.customer : : : +- CometBroadcastExchange : : : +- CometFilter : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -45,10 +45,10 @@ CometColumnarToRow : : : +- BroadcastExchange : : : +- CometColumnarToRow : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometFilter : +- CometHashAggregate @@ -60,14 +60,14 @@ CometColumnarToRow : : +- CometBroadcastHashJoin : : :- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : : +- CometNativeScan parquet spark_catalog.default.customer : : +- CometBroadcastExchange : : +- CometFilter : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : : +- ReusedSubquery : +- CometBroadcastExchange : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometHashAggregate +- CometExchange @@ -78,13 +78,13 @@ CometColumnarToRow : +- CometBroadcastHashJoin : :- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer + : : +- CometNativeScan parquet spark_catalog.default.customer : +- CometBroadcastExchange : +- CometFilter : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales : +- ReusedSubquery +- CometBroadcastExchange +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 79 out of 85 eligible operators (92%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt index 72e39422e0..6411ab4b43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -27,10 +27,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] @@ -51,10 +51,10 @@ WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] @@ -66,7 +66,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt index 6925eb301e..e7f56d9c92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt @@ -26,15 +26,15 @@ : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -51,7 +51,7 @@ : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -68,7 +68,7 @@ : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -92,7 +92,7 @@ : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -137,7 +137,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -78,7 +78,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -213,10 +213,10 @@ BroadcastExchange (42) +- * CometColumnarToRow (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) + +- CometNativeScan parquet spark_catalog.default.date_dim (38) -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(38) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt index 46c47555a8..0a00a8c60b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt @@ -22,15 +22,15 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item :- CometHashAggregate : +- CometExchange : +- CometHashAggregate @@ -48,15 +48,15 @@ TakeOrderedAndProject : : : +- CometColumnarToRow : : : +- CometProject : : : +- CometFilter - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : +- CometBroadcastExchange : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometHashAggregate +- CometExchange +- CometHashAggregate @@ -74,14 +74,14 @@ TakeOrderedAndProject : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + +- CometNativeScan parquet spark_catalog.default.item Comet accelerated 72 out of 81 eligible operators (88%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt index 11e3c03d80..fd3b3cf399 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt @@ -28,15 +28,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #6 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] CometExchange [i_category] #7 CometHashAggregate [total_sum] [i_category,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt index 96a0ec392e..62d0895b96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt @@ -19,11 +19,11 @@ : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometNativeScan parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + +- CometNativeScan parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(26) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt index 3dbaf2e346..e9f812d706 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt @@ -20,14 +20,14 @@ CometColumnarToRow : : +- CometColumnarToRow : : +- CometProject : : +- CometFilter - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim : +- CometBroadcastExchange : +- CometProject : +- CometFilter - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item + : +- CometNativeScan parquet spark_catalog.default.item +- CometBroadcastExchange +- CometProject +- CometFilter - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt index 46191f59cd..af593e1269 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt @@ -28,12 +28,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] From 749f796655c48c75d599cea71709ca40ec6cb47d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Dec 2025 10:04:17 -0700 Subject: [PATCH 7/8] fix --- .../scala/org/apache/comet/rules/CometScanRule.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 9c8cc3f8de..d125e482d3 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -20,17 +20,15 @@ package org.apache.comet.rules import java.net.URI - import scala.collection.mutable import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ - import org.apache.hadoop.conf.Configuration import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, GenericInternalRow, PlanExpression} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.util.{sideBySide, ArrayBasedMapData, GenericArrayData, MetadataColumnHelper} +import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MetadataColumnHelper, sideBySide} import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues import org.apache.spark.sql.comet.{CometBatchScanExec, CometScanExec} import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} @@ -39,10 +37,9 @@ import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ - import org.apache.comet.{CometConf, CometNativeException, DataTypeSupport} import org.apache.comet.CometConf._ -import org.apache.comet.CometSparkSessionExtensions.{isCometLoaded, withInfo, withInfos} +import org.apache.comet.CometSparkSessionExtensions.{isCometLoaded, isSpark40Plus, withInfo, withInfos} import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.iceberg.{CometIcebergNativeScanMetadata, IcebergReflection} import org.apache.comet.objectstore.NativeConfig @@ -185,6 +182,10 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com scanExec: FileSourceScanExec, r: HadoopFsRelation, hadoopConf: Configuration): Option[SparkPlan] = { + if (isSpark40Plus) { + // there are still issues with Spark 4 support + return None + } if (!CometNativeScan.isSupported(scanExec)) { return None } From d281304cdaaf930309b352e239988354bac5151a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Dec 2025 10:12:47 -0700 Subject: [PATCH 8/8] format --- .../main/scala/org/apache/comet/rules/CometScanRule.scala | 5 ++++- .../src/test/scala/org/apache/comet/CometFuzzTestBase.scala | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index d125e482d3..e626240156 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -20,15 +20,17 @@ package org.apache.comet.rules import java.net.URI + import scala.collection.mutable import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ + import org.apache.hadoop.conf.Configuration import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, GenericInternalRow, PlanExpression} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MetadataColumnHelper, sideBySide} +import org.apache.spark.sql.catalyst.util.{sideBySide, ArrayBasedMapData, GenericArrayData, MetadataColumnHelper} import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues import org.apache.spark.sql.comet.{CometBatchScanExec, CometScanExec} import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} @@ -37,6 +39,7 @@ import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ + import org.apache.comet.{CometConf, CometNativeException, DataTypeSupport} import org.apache.comet.CometConf._ import org.apache.comet.CometSparkSessionExtensions.{isCometLoaded, isSpark40Plus, withInfo, withInfos} diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala index ca4d068796..74858ed614 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala @@ -39,10 +39,9 @@ import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGener class CometFuzzTestBase extends CometTestBase with AdaptiveSparkPlanHelper { - /** Path for generated data files */ var filename: String = null - /** Path for generated data files with deeply nested complex types */ + /** Filename for data file with deeply nested complex types */ var complexTypesFilename: String = null /**