diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 69bce75559..7022285f02 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -478,29 +478,33 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] with Com false } - // Check for unsupported transform functions in residual expressions - // iceberg-rust can only handle identity transforms in residuals; all other transforms - // (truncate, bucket, year, month, day, hour) must fall back to Spark + // Check for transform functions in residual expressions + // Non-identity transforms (truncate, bucket, year, month, day, hour) in residuals + // are now supported - they skip row-group filtering and are handled post-scan by CometFilter. + // This is less optimal than row-group filtering but still allows native execution. val transformFunctionsSupported = try { IcebergReflection.findNonIdentityTransformInResiduals(metadata.tasks) match { case Some(transformType) => - // Found unsupported transform - fallbackReasons += - s"Iceberg transform function '$transformType' in residual expression " + - "is not yet supported by iceberg-rust. " + - "Only identity transforms are supported." - false + // Found non-identity transform - log info and continue with native scan + // Row-group filtering will skip these predicates, but post-scan filtering will apply + logInfo( + s"Iceberg residual contains transform '$transformType' - " + + "row-group filtering will skip this predicate, " + + "post-scan filtering by CometFilter will apply instead.") + true // Allow native execution case None => - // No unsupported transforms found - safe to use native execution + // No non-identity transforms - optimal row-group filtering will apply true } } catch { case e: Exception => - // Reflection failure - cannot verify safety, must fall back - fallbackReasons += "Iceberg reflection failure: Could not check for " + - s"transform functions in residuals: ${e.getMessage}" - false + // Reflection failure - log warning but allow native execution + // The predicate conversion will handle unsupported cases gracefully + logWarning( + s"Could not check for transform functions in residuals: ${e.getMessage}. " + + "Continuing with native scan.") + true } // Check for unsupported struct types in delete files