Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala
Original file line number Diff line number Diff line change
Expand Up @@ -478,29 +478,33 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com
false
}

// Check for unsupported transform functions in residual expressions
// iceberg-rust can only handle identity transforms in residuals; all other transforms
// (truncate, bucket, year, month, day, hour) must fall back to Spark
// Check for transform functions in residual expressions
// Non-identity transforms (truncate, bucket, year, month, day, hour) in residuals
// are now supported - they skip row-group filtering and are handled post-scan by CometFilter.
// This is less optimal than row-group filtering but still allows native execution.
val transformFunctionsSupported =
try {
IcebergReflection.findNonIdentityTransformInResiduals(metadata.tasks) match {
case Some(transformType) =>
// Found unsupported transform
fallbackReasons +=
s"Iceberg transform function '$transformType' in residual expression " +
"is not yet supported by iceberg-rust. " +
"Only identity transforms are supported."
false
// Found non-identity transform - log info and continue with native scan
// Row-group filtering will skip these predicates, but post-scan filtering will apply
logInfo(
s"Iceberg residual contains transform '$transformType' - " +
"row-group filtering will skip this predicate, " +
"post-scan filtering by CometFilter will apply instead.")
true // Allow native execution
case None =>
// No unsupported transforms found - safe to use native execution
// No non-identity transforms - optimal row-group filtering will apply
true
}
} catch {
case e: Exception =>
// Reflection failure - cannot verify safety, must fall back
fallbackReasons += "Iceberg reflection failure: Could not check for " +
s"transform functions in residuals: ${e.getMessage}"
false
// Reflection failure - log warning but allow native execution
// The predicate conversion will handle unsupported cases gracefully
logWarning(
s"Could not check for transform functions in residuals: ${e.getMessage}. " +
"Continuing with native scan.")
true
}

// Check for unsupported struct types in delete files
Expand Down
Loading