diff --git a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala index bb0466b4eb..e6e32ebbc5 100644 --- a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala @@ -205,8 +205,6 @@ object CometAverage extends CometAggregateExpressionSerde[Average] { object CometSum extends CometAggregateExpressionSerde[Sum] { - override def getIncompatibleReasons(): Seq[String] = Seq("Falls back to Spark in ANSI mode.") - override def convert( aggExpr: AggregateExpression, sum: Sum, diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala index 968fe8cd69..4c818f24c6 100644 --- a/spark/src/main/scala/org/apache/comet/serde/strings.scala +++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala @@ -33,7 +33,7 @@ import org.apache.comet.serde.QueryPlanSerde.{createBinaryExpr, exprToProtoInter object CometStringRepeat extends CometExpressionSerde[StringRepeat] { - override def getCompatibleNotes(): Seq[String] = Seq( + override def getIncompatibleReasons(): Seq[String] = Seq( "A negative argument for the number of times to repeat throws an exception" + " instead of returning an empty string as Spark does") diff --git a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala index e5eeb5b848..b561c24787 100644 --- a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala @@ -29,12 +29,27 @@ import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithIn // https://github.com/apache/datafusion/issues/16594 object CometFromUnixTime extends CometExpressionSerde[FromUnixTime] { + override def getUnsupportedReasons(): Seq[String] = Seq( + "Only supports the default datetime format pattern `yyyy-MM-dd HH:mm:ss`") + override def getIncompatibleReasons(): Seq[String] = Seq( - "Only supports the default datetime format pattern `yyyy-MM-dd HH:mm:ss`." + - " DataFusion's valid timestamp range differs from Spark" + + "DataFusion's valid timestamp range differs from Spark" + " (https://github.com/apache/datafusion/issues/16594)") - override def getSupportLevel(expr: FromUnixTime): SupportLevel = Incompatible(None) + override def getSupportLevel(expr: FromUnixTime): SupportLevel = { + expr.format match { + case Literal(fmt, _) => + val formatStr = fmt.toString + val defaultPattern = TimestampFormatter.defaultPattern + if (formatStr == defaultPattern) { + Incompatible(None) + } else { + Unsupported(Some(s"Datetime pattern format: $formatStr is unsupported")) + } + case _ => + Unsupported(Some("Non-literal datetime pattern format is unsupported")) + } + } override def convert( expr: FromUnixTime, @@ -48,10 +63,7 @@ object CometFromUnixTime extends CometExpressionSerde[FromUnixTime] { val formatExpr = exprToProtoInternal(Literal("%Y-%m-%d %H:%M:%S"), inputs, binding) val timeZone = exprToProtoInternal(Literal(expr.timeZoneId.orNull), inputs, binding) - if (expr.format != Literal(TimestampFormatter.defaultPattern)) { - withInfo(expr, "Datetime pattern format is unsupported") - None - } else if (secExpr.isDefined && formatExpr.isDefined) { + if (secExpr.isDefined && formatExpr.isDefined) { val timestampExpr = scalarFunctionExprToProto("from_unixtime", Seq(secExpr, timeZone): _*) val optExpr = scalarFunctionExprToProto("to_char", Seq(timestampExpr, formatExpr): _*) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql index a7b0960570..47c76d67aa 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql @@ -24,12 +24,12 @@ INSERT INTO test_from_unix_time VALUES (0), (1718451045), (-1), (NULL), (2147483 query expect_fallback(not fully compatible with Spark) SELECT from_unixtime(t) FROM test_from_unix_time -query expect_fallback(not fully compatible with Spark) +query expect_fallback(Datetime pattern format: yyyy-MM-dd is unsupported) SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time -- literal arguments query expect_fallback(not fully compatible with Spark) SELECT from_unixtime(0) -query expect_fallback(not fully compatible with Spark) +query expect_fallback(Datetime pattern format: yyyy-MM-dd is unsupported) SELECT from_unixtime(1718451045, 'yyyy-MM-dd')