Skip to content

Commit 4cb98f9

Browse files
mbutrovichmbutrovich
committed
Update spark diffs, new golden files.
1 parent 416e5b8 commit 4cb98f9

331 files changed

Lines changed: 7866 additions & 9211 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎dev/diffs/3.4.3.diff‎

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -417,18 +417,26 @@ index daef11ae4d6..9f3cc9181f2 100644
417417
assert(exchanges.size == 2)
418418
}
419419
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
420-
index f33432ddb6f..8ec7285ed84 100644
420+
index f33432ddb6f..579a9c271de 100644
421421
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
422422
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
423423
@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
424424
import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
425425
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
426426
import org.apache.spark.sql.catalyst.plans.ExistenceJoin
427-
+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec}
427+
+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec, CometSubqueryBroadcastExec}
428428
import org.apache.spark.sql.connector.catalog.{InMemoryTableCatalog, InMemoryTableWithV2FilterCatalog}
429429
import org.apache.spark.sql.execution._
430430
import org.apache.spark.sql.execution.adaptive._
431-
@@ -262,6 +263,12 @@ abstract class DynamicPartitionPruningSuiteBase
431+
@@ -193,6 +194,7 @@ abstract class DynamicPartitionPruningSuiteBase
432+
}
433+
val subqueryBroadcast = dpExprs.collect {
434+
case InSubqueryExec(_, b: SubqueryBroadcastExec, _, _, _, _) => b
435+
+ case InSubqueryExec(_, b: CometSubqueryBroadcastExec, _, _, _, _) => b
436+
}
437+
438+
val hasFilter = if (withSubquery) "Should" else "Shouldn't"
439+
@@ -262,6 +264,12 @@ abstract class DynamicPartitionPruningSuiteBase
432440
case s: BatchScanExec => s.runtimeFilters.collect {
433441
case d: DynamicPruningExpression => d.child
434442
}
@@ -441,7 +449,7 @@ index f33432ddb6f..8ec7285ed84 100644
441449
case _ => Nil
442450
}
443451
}
444-
@@ -1027,7 +1034,8 @@ abstract class DynamicPartitionPruningSuiteBase
452+
@@ -1027,7 +1035,8 @@ abstract class DynamicPartitionPruningSuiteBase
445453
}
446454
}
447455

@@ -451,7 +459,7 @@ index f33432ddb6f..8ec7285ed84 100644
451459
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
452460
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
453461
withTable("large", "dimTwo", "dimThree") {
454-
@@ -1215,7 +1223,8 @@ abstract class DynamicPartitionPruningSuiteBase
462+
@@ -1215,7 +1224,8 @@ abstract class DynamicPartitionPruningSuiteBase
455463
}
456464

457465
test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
@@ -461,7 +469,7 @@ index f33432ddb6f..8ec7285ed84 100644
461469
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
462470
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
463471
val df = sql(
464-
@@ -1423,7 +1432,8 @@ abstract class DynamicPartitionPruningSuiteBase
472+
@@ -1423,7 +1433,8 @@ abstract class DynamicPartitionPruningSuiteBase
465473
}
466474
}
467475

@@ -471,7 +479,7 @@ index f33432ddb6f..8ec7285ed84 100644
471479
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
472480
val df = sql(
473481
""" WITH v as (
474-
@@ -1698,7 +1708,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
482+
@@ -1698,7 +1709,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
475483
* Check the static scan metrics with and without DPP
476484
*/
477485
test("static scan metrics",
@@ -481,7 +489,7 @@ index f33432ddb6f..8ec7285ed84 100644
481489
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
482490
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
483491
SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
484-
@@ -1729,6 +1740,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
492+
@@ -1729,6 +1741,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
485493
case s: BatchScanExec =>
486494
// we use f1 col for v2 tables due to schema pruning
487495
s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))
@@ -1297,7 +1305,7 @@ index 4b3d3a4b805..56e1e0e6f16 100644
12971305

12981306
setupTestData()
12991307
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
1300-
index 9e9d717db3b..ec73082f458 100644
1308+
index 9e9d717db3b..ad41c8f3901 100644
13011309
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
13021310
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
13031311
@@ -17,7 +17,10 @@
@@ -1324,17 +1332,7 @@ index 9e9d717db3b..ec73082f458 100644
13241332
assert(actual == expected)
13251333
}
13261334
}
1327-
@@ -112,7 +118,8 @@ abstract class RemoveRedundantProjectsSuiteBase
1328-
assertProjectExec(query, 1, 3)
1329-
}
1330-
1331-
- test("join with ordering requirement") {
1332-
+ test("join with ordering requirement",
1333-
+ IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
1334-
val query = "select * from (select key, a, c, b from testView) as t1 join " +
1335-
"(select key, a, b, c from testView) as t2 on t1.key = t2.key where t2.a > 50"
1336-
assertProjectExec(query, 2, 2)
1337-
@@ -134,12 +141,21 @@ abstract class RemoveRedundantProjectsSuiteBase
1335+
@@ -134,12 +140,21 @@ abstract class RemoveRedundantProjectsSuiteBase
13381336
val df = data.selectExpr("a", "b", "key", "explode(array(key, a, b)) as d").filter("d > 0")
13391337
df.collect()
13401338
val plan = df.queryExecution.executedPlan
@@ -1361,7 +1359,7 @@ index 9e9d717db3b..ec73082f458 100644
13611359
case g @ GenerateExec(_, requiredChildOutput, _, _, child) =>
13621360
g.copy(requiredChildOutput = requiredChildOutput.reverse,
13631361
child = ProjectExec(requiredChildOutput.reverse, child))
1364-
@@ -151,6 +167,7 @@ abstract class RemoveRedundantProjectsSuiteBase
1362+
@@ -151,6 +166,7 @@ abstract class RemoveRedundantProjectsSuiteBase
13651363
// The manually added ProjectExec node shouldn't be removed.
13661364
assert(collectWithSubqueries(newExecutedPlan) {
13671365
case p: ProjectExec => p

‎dev/diffs/3.5.8.diff‎

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -398,18 +398,26 @@ index c4fb4fa943c..a04b23870a8 100644
398398
assert(exchanges.size == 2)
399399
}
400400
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
401-
index f33432ddb6f..8ec7285ed84 100644
401+
index f33432ddb6f..579a9c271de 100644
402402
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
403403
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
404404
@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
405405
import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
406406
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
407407
import org.apache.spark.sql.catalyst.plans.ExistenceJoin
408-
+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec}
408+
+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec, CometSubqueryBroadcastExec}
409409
import org.apache.spark.sql.connector.catalog.{InMemoryTableCatalog, InMemoryTableWithV2FilterCatalog}
410410
import org.apache.spark.sql.execution._
411411
import org.apache.spark.sql.execution.adaptive._
412-
@@ -262,6 +263,12 @@ abstract class DynamicPartitionPruningSuiteBase
412+
@@ -193,6 +194,7 @@ abstract class DynamicPartitionPruningSuiteBase
413+
}
414+
val subqueryBroadcast = dpExprs.collect {
415+
case InSubqueryExec(_, b: SubqueryBroadcastExec, _, _, _, _) => b
416+
+ case InSubqueryExec(_, b: CometSubqueryBroadcastExec, _, _, _, _) => b
417+
}
418+
419+
val hasFilter = if (withSubquery) "Should" else "Shouldn't"
420+
@@ -262,6 +264,12 @@ abstract class DynamicPartitionPruningSuiteBase
413421
case s: BatchScanExec => s.runtimeFilters.collect {
414422
case d: DynamicPruningExpression => d.child
415423
}
@@ -422,7 +430,7 @@ index f33432ddb6f..8ec7285ed84 100644
422430
case _ => Nil
423431
}
424432
}
425-
@@ -1027,7 +1034,8 @@ abstract class DynamicPartitionPruningSuiteBase
433+
@@ -1027,7 +1035,8 @@ abstract class DynamicPartitionPruningSuiteBase
426434
}
427435
}
428436

@@ -432,7 +440,7 @@ index f33432ddb6f..8ec7285ed84 100644
432440
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
433441
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
434442
withTable("large", "dimTwo", "dimThree") {
435-
@@ -1215,7 +1223,8 @@ abstract class DynamicPartitionPruningSuiteBase
443+
@@ -1215,7 +1224,8 @@ abstract class DynamicPartitionPruningSuiteBase
436444
}
437445

438446
test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
@@ -442,7 +450,7 @@ index f33432ddb6f..8ec7285ed84 100644
442450
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
443451
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
444452
val df = sql(
445-
@@ -1423,7 +1432,8 @@ abstract class DynamicPartitionPruningSuiteBase
453+
@@ -1423,7 +1433,8 @@ abstract class DynamicPartitionPruningSuiteBase
446454
}
447455
}
448456

@@ -452,7 +460,7 @@ index f33432ddb6f..8ec7285ed84 100644
452460
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
453461
val df = sql(
454462
""" WITH v as (
455-
@@ -1698,7 +1708,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
463+
@@ -1698,7 +1709,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
456464
* Check the static scan metrics with and without DPP
457465
*/
458466
test("static scan metrics",
@@ -462,7 +470,7 @@ index f33432ddb6f..8ec7285ed84 100644
462470
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
463471
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
464472
SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
465-
@@ -1729,6 +1740,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
473+
@@ -1729,6 +1741,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
466474
case s: BatchScanExec =>
467475
// we use f1 col for v2 tables due to schema pruning
468476
s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))
@@ -1248,17 +1256,16 @@ index de24b8c82b0..1f835481290 100644
12481256

12491257
setupTestData()
12501258
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
1251-
index 9e9d717db3b..cdd1042a880 100644
1259+
index 9e9d717db3b..73de2b84938 100644
12521260
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
12531261
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
12541262
@@ -17,7 +17,10 @@
12551263

12561264
package org.apache.spark.sql.execution
12571265

1258-
-import org.apache.spark.sql.{DataFrame, QueryTest, Row}
12591266
+import org.apache.comet.CometConf
12601267
+
1261-
+import org.apache.spark.sql.{DataFrame, IgnoreCometNativeDataFusion, QueryTest, Row}
1268+
import org.apache.spark.sql.{DataFrame, QueryTest, Row}
12621269
+import org.apache.spark.sql.comet.CometProjectExec
12631270
import org.apache.spark.sql.connector.SimpleWritableDataSource
12641271
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
@@ -1275,17 +1282,7 @@ index 9e9d717db3b..cdd1042a880 100644
12751282
assert(actual == expected)
12761283
}
12771284
}
1278-
@@ -112,7 +118,8 @@ abstract class RemoveRedundantProjectsSuiteBase
1279-
assertProjectExec(query, 1, 3)
1280-
}
1281-
1282-
- test("join with ordering requirement") {
1283-
+ test("join with ordering requirement",
1284-
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4014")) {
1285-
val query = "select * from (select key, a, c, b from testView) as t1 join " +
1286-
"(select key, a, b, c from testView) as t2 on t1.key = t2.key where t2.a > 50"
1287-
assertProjectExec(query, 2, 2)
1288-
@@ -134,12 +141,26 @@ abstract class RemoveRedundantProjectsSuiteBase
1285+
@@ -134,12 +140,26 @@ abstract class RemoveRedundantProjectsSuiteBase
12891286
val df = data.selectExpr("a", "b", "key", "explode(array(key, a, b)) as d").filter("d > 0")
12901287
df.collect()
12911288
val plan = df.queryExecution.executedPlan
@@ -1314,7 +1311,7 @@ index 9e9d717db3b..cdd1042a880 100644
13141311
case g @ GenerateExec(_, requiredChildOutput, _, _, child) =>
13151312
g.copy(requiredChildOutput = requiredChildOutput.reverse,
13161313
child = ProjectExec(requiredChildOutput.reverse, child))
1317-
@@ -151,6 +172,7 @@ abstract class RemoveRedundantProjectsSuiteBase
1314+
@@ -151,6 +171,7 @@ abstract class RemoveRedundantProjectsSuiteBase
13181315
// The manually added ProjectExec node shouldn't be removed.
13191316
assert(collectWithSubqueries(newExecutedPlan) {
13201317
case p: ProjectExec => p

‎dev/diffs/4.0.1.diff‎

Lines changed: 21 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -589,18 +589,26 @@ index 81713c777bc..b5f92ed9742 100644
589589
assert(exchanges.size == 2)
590590
}
591591
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
592-
index 2c24cc7d570..697ea7f8a60 100644
592+
index 2c24cc7d570..fd3c85eef87 100644
593593
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
594594
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
595595
@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
596596
import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
597597
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
598598
import org.apache.spark.sql.catalyst.plans.ExistenceJoin
599-
+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec}
599+
+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec, CometSubqueryBroadcastExec}
600600
import org.apache.spark.sql.connector.catalog.{InMemoryTableCatalog, InMemoryTableWithV2FilterCatalog}
601601
import org.apache.spark.sql.execution._
602602
import org.apache.spark.sql.execution.adaptive._
603-
@@ -262,6 +263,12 @@ abstract class DynamicPartitionPruningSuiteBase
603+
@@ -193,6 +194,7 @@ abstract class DynamicPartitionPruningSuiteBase
604+
}
605+
val subqueryBroadcast = dpExprs.collect {
606+
case InSubqueryExec(_, b: SubqueryBroadcastExec, _, _, _, _) => b
607+
+ case InSubqueryExec(_, b: CometSubqueryBroadcastExec, _, _, _, _) => b
608+
}
609+
610+
val hasFilter = if (withSubquery) "Should" else "Shouldn't"
611+
@@ -262,6 +264,12 @@ abstract class DynamicPartitionPruningSuiteBase
604612
case s: BatchScanExec => s.runtimeFilters.collect {
605613
case d: DynamicPruningExpression => d.child
606614
}
@@ -613,7 +621,7 @@ index 2c24cc7d570..697ea7f8a60 100644
613621
case _ => Nil
614622
}
615623
}
616-
@@ -1027,7 +1034,8 @@ abstract class DynamicPartitionPruningSuiteBase
624+
@@ -1027,7 +1035,8 @@ abstract class DynamicPartitionPruningSuiteBase
617625
}
618626
}
619627

@@ -623,7 +631,7 @@ index 2c24cc7d570..697ea7f8a60 100644
623631
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
624632
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
625633
withTable("large", "dimTwo", "dimThree") {
626-
@@ -1151,7 +1159,8 @@ abstract class DynamicPartitionPruningSuiteBase
634+
@@ -1151,7 +1160,8 @@ abstract class DynamicPartitionPruningSuiteBase
627635
}
628636
}
629637

@@ -633,7 +641,7 @@ index 2c24cc7d570..697ea7f8a60 100644
633641
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
634642
SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName,
635643
SQLConf.ANSI_ENABLED.key -> "false" // ANSI mode doesn't support "String + String"
636-
@@ -1215,7 +1224,8 @@ abstract class DynamicPartitionPruningSuiteBase
644+
@@ -1215,7 +1225,8 @@ abstract class DynamicPartitionPruningSuiteBase
637645
}
638646

639647
test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
@@ -643,15 +651,15 @@ index 2c24cc7d570..697ea7f8a60 100644
643651
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
644652
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
645653
val df = sql(
646-
@@ -1330,6 +1340,7 @@ abstract class DynamicPartitionPruningSuiteBase
654+
@@ -1330,6 +1341,7 @@ abstract class DynamicPartitionPruningSuiteBase
647655
}
648656

649657
test("Subquery reuse across the whole plan",
650658
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3321"),
651659
DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
652660
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
653661
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
654-
@@ -1424,7 +1435,8 @@ abstract class DynamicPartitionPruningSuiteBase
662+
@@ -1424,7 +1436,8 @@ abstract class DynamicPartitionPruningSuiteBase
655663
}
656664
}
657665

@@ -661,7 +669,7 @@ index 2c24cc7d570..697ea7f8a60 100644
661669
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
662670
val df = sql(
663671
""" WITH v as (
664-
@@ -1699,7 +1711,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
672+
@@ -1699,7 +1712,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
665673
* Check the static scan metrics with and without DPP
666674
*/
667675
test("static scan metrics",
@@ -671,7 +679,7 @@ index 2c24cc7d570..697ea7f8a60 100644
671679
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
672680
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
673681
SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
674-
@@ -1730,6 +1743,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
682+
@@ -1730,6 +1744,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
675683
case s: BatchScanExec =>
676684
// we use f1 col for v2 tables due to schema pruning
677685
s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))
@@ -1755,7 +1763,7 @@ index 47d5ff67b84..8dc8f65d4b1 100644
17551763
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
17561764
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
17571765
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
1758-
index b5bac8079c4..9420dbdb936 100644
1766+
index b5bac8079c4..1606dd057e0 100644
17591767
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
17601768
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
17611769
@@ -17,7 +17,10 @@
@@ -1782,17 +1790,7 @@ index b5bac8079c4..9420dbdb936 100644
17821790
assert(actual == expected)
17831791
}
17841792
}
1785-
@@ -112,7 +118,8 @@ abstract class RemoveRedundantProjectsSuiteBase
1786-
assertProjectExec(query, 1, 3)
1787-
}
1788-
1789-
- test("join with ordering requirement") {
1790-
+ test("join with ordering requirement",
1791-
+ IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
1792-
val query = "select * from (select key, a, c, b from testView) as t1 join " +
1793-
"(select key, a, b, c from testView) as t2 on t1.key = t2.key where t2.a > 50"
1794-
assertProjectExec(query, 2, 2)
1795-
@@ -134,12 +141,25 @@ abstract class RemoveRedundantProjectsSuiteBase
1793+
@@ -134,12 +140,25 @@ abstract class RemoveRedundantProjectsSuiteBase
17961794
val df = data.selectExpr("a", "b", "key", "explode(array(key, a, b)) as d").filter("d > 0")
17971795
df.collect()
17981796
val plan = df.queryExecution.executedPlan
@@ -1820,7 +1818,7 @@ index b5bac8079c4..9420dbdb936 100644
18201818
case g @ GenerateExec(_, requiredChildOutput, _, _, child) =>
18211819
g.copy(requiredChildOutput = requiredChildOutput.reverse,
18221820
child = ProjectExec(requiredChildOutput.reverse, child))
1823-
@@ -151,6 +171,7 @@ abstract class RemoveRedundantProjectsSuiteBase
1821+
@@ -151,6 +170,7 @@ abstract class RemoveRedundantProjectsSuiteBase
18241822
// The manually added ProjectExec node shouldn't be removed.
18251823
assert(collectWithSubqueries(newExecutedPlan) {
18261824
case p: ProjectExec => p

0 commit comments

Comments
 (0)