rebase_main

Author: coderfender

dev/diffs/4.0.1.diff

@@ -1,5 +1,5 @@
 diff --git a/pom.xml b/pom.xml
-index 443d46a4302..9880184c4e3 100644
+index 22922143fc3..7c56e5e8641 100644
 --- a/pom.xml
 +++ b/pom.xml
 @@ -148,6 +148,8 @@
@@ -38,7 +38,7 @@ index 443d46a4302..9880184c4e3 100644
        <dependency>
          <groupId>org.apache.datasketches</groupId>
 diff --git a/sql/core/pom.xml b/sql/core/pom.xml
-index 56aad01023d..ad9d6a973e4 100644
+index dcf6223a98b..0458a5bb640 100644
 --- a/sql/core/pom.xml
 +++ b/sql/core/pom.xml
 @@ -90,6 +90,10 @@
@@ -373,7 +373,7 @@ index 1f8c5822e7d..b7de4e28813 100644
  WITH t(c1) AS (SELECT replace(listagg(DISTINCT col1 COLLATE unicode_rtrim) COLLATE utf8_binary, ' ', '') FROM (VALUES ('xbc  '), ('xbc '), ('a'), ('xbc'))) SELECT len(c1), regexp_count(c1, 'a'), regexp_count(c1, 'xbc') FROM t
  -- !query schema
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
-index e0ad3feda3a..8fc1ba8e10d 100644
+index 0f42502f1d9..f616024a9c2 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
 @@ -39,7 +39,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants
@@ -421,7 +421,7 @@ index e0ad3feda3a..8fc1ba8e10d 100644
 
        withTempView("t0", "t1", "t2") {
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
-index 6ce0a657d5b..02e352d13b0 100644
+index 9db406ff12f..abbc91f5c11 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
 @@ -30,7 +30,7 @@ import org.apache.spark.sql.errors.DataTypeErrors.toSQLId
@@ -843,7 +843,7 @@ index 53e47f428c3..a55d8f0c161 100644
      assert(shuffleMergeJoins.size == 1)
    }
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-index 41f2e5c9a40..8f0dd909492 100644
+index aaac0ebc9aa..fbef0774d46 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
 @@ -29,7 +29,8 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
@@ -1041,7 +1041,7 @@ index 41f2e5c9a40..8f0dd909492 100644
        checkAnswer(fullJoinDF, Row(100))
      }
    }
-@@ -1586,6 +1615,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1614,6 +1643,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
            Seq(semiJoinDF, antiJoinDF).foreach { df =>
              assert(collect(df.queryExecution.executedPlan) {
                case j: ShuffledHashJoinExec if j.ignoreDuplicatedKey == ignoreDuplicatedKey => true
@@ -1051,7 +1051,7 @@ index 41f2e5c9a40..8f0dd909492 100644
              }.size == 1)
            }
        }
-@@ -1630,14 +1662,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
+@@ -1658,14 +1690,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
 
    test("SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)") {
      def check(plan: SparkPlan): Unit = {
@@ -1074,7 +1074,7 @@ index 41f2e5c9a40..8f0dd909492 100644
      }
      dupStreamSideColTest("SHUFFLE_HASH", check)
    }
-@@ -1773,7 +1811,8 @@ class ThreadLeakInSortMergeJoinSuite
+@@ -1801,7 +1839,8 @@ class ThreadLeakInSortMergeJoinSuite
        sparkConf.set(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD, 20))
    }
 
@@ -1111,7 +1111,7 @@ index 818c7ec65b1..0d333807b70 100644
        createTable("t1", Seq("i"), Seq("int"))
        createTable("t2", Seq("i"), Seq("int"))
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
-index b3fce19979e..67edf5eb91c 100644
+index f294ff81021..8a3b818ee94 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
 @@ -1524,7 +1524,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
@@ -1427,19 +1427,21 @@ index 11e9547dfc5..be9ae40ab3d 100644
      val t2 = "T_2"
 
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
-index 031af8200d7..bb5acd74ad2 100644
+index 77ed8b411e3..18b51f0c939 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
-@@ -17,7 +17,7 @@
+@@ -17,9 +17,9 @@
 
  package org.apache.spark.sql.collation
 
 -import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+ import org.apache.spark.sql.catalyst.expressions.AttributeReference
+ import org.apache.spark.sql.catalyst.plans.logical.Project
 +import org.apache.spark.sql.{AnalysisException, DataFrame, IgnoreComet, QueryTest, Row}
  import org.apache.spark.sql.catalyst.util.CollationFactory
  import org.apache.spark.sql.connector.DatasourceV2SQLBase
  import org.apache.spark.sql.test.SharedSparkSession
-@@ -184,7 +184,7 @@ abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSessi
+@@ -220,7 +220,7 @@ abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSessi
      }
    }
 
@@ -1537,7 +1539,7 @@ index 2a0ab21ddb0..e8a5a891105 100644
          } finally {
            spark.listenerManager.unregister(listener)
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-index c24f52bd930..ce4d4c4a694 100644
+index c73e8e16fbb..88cd0d47da3 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
 @@ -20,10 +20,11 @@ import java.sql.Timestamp
@@ -1614,7 +1616,7 @@ index f62e092138a..c0404bfe85e 100644
 
    before {
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
-index 46f2502582c..1f4c8f2201b 100644
+index 04d33ecd3d5..450df347297 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
 @@ -31,7 +31,7 @@ import org.mockito.Mockito.{mock, spy, when}
@@ -1755,7 +1757,7 @@ index 1400ee25f43..5b016c3f9c5 100644
 
    setupTestData()
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
-index a6490243708..c61af08d4cf 100644
+index 47d5ff67b84..8dc8f65d4b1 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
 @@ -20,7 +20,7 @@ import scala.collection.mutable
@@ -1767,7 +1769,7 @@ index a6490243708..c61af08d4cf 100644
  import org.apache.spark.sql.catalyst.{QueryPlanningTracker, QueryPlanningTrackerCallback, TableIdentifier}
  import org.apache.spark.sql.catalyst.analysis.{CurrentNamespace, UnresolvedFunction, UnresolvedRelation}
  import org.apache.spark.sql.catalyst.expressions.{Alias, UnsafeRow}
-@@ -385,7 +385,7 @@ class QueryExecutionSuite extends SharedSparkSession {
+@@ -400,7 +400,7 @@ class QueryExecutionSuite extends SharedSparkSession {
      }
    }
 
@@ -2868,10 +2870,10 @@ index 6080a5e8e4b..9aa8f49a62b 100644
          case _ => assert(false, "Can not match ParquetTable in the query.")
        }
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-index 22839d3f0d2..812d642f15d 100644
+index 4474ec1fd42..d0366be3597 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-@@ -1334,7 +1334,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
+@@ -1344,7 +1344,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
      }
    }
 
@@ -2881,7 +2883,7 @@ index 22839d3f0d2..812d642f15d 100644
      withAllParquetReaders {
        checkAnswer(
          // "fruit" column in this file is encoded using DELTA_LENGTH_BYTE_ARRAY.
-@@ -1543,7 +1544,9 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
+@@ -1553,7 +1554,9 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
      }
    }