Skip to content

fix: enable ParquetFilterSuite tests for native_datafusion #3698

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
andygrove wants to merge 2 commits into from
Closed

fix: enable ParquetFilterSuite tests for native_datafusion #3698

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ec0f453
fix: enable ParquetFilterSuite tests for native_datafusion (#3320)
andygrove Mar 14, 2026
ce7fc27
fix: merge apache/main into fix/enable-native-datafusion-parquet-filt…
andygrove Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 11 additions & 29 deletions dev/diffs/3.5.8.diff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -965,7 +965,7 @@ index 3cf2bfd17ab..49728c35c42 100644
SQLConf.ANSI_ENABLED.key -> "true") {
withTable("t") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
index fa1a64460fc..1d2e215d6a3 100644
index fa1a64460fc..134f0db1fb8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -17,6 +17,8 @@
Expand Down Expand Up @@ -2038,7 +2038,7 @@ index 07e2849ce6f..3e73645b638 100644
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 8e88049f51e..49f2001dc6b 100644
index 8e88049f51e..99561d9e7c7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
Expand All @@ -2064,17 +2064,7 @@ index 8e88049f51e..49f2001dc6b 100644
import testImplicits._

withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -1548,7 +1553,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
}
}

- test("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") {
+ test("SPARK-31026: Parquet predicate pushdown for fields having dots in the names",
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3320")) {
import testImplicits._

withAllParquetReaders {
@@ -1580,13 +1586,18 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1580,7 +1585,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
// than the total length but should not be a single record.
// Note that, if record level filtering is enabled, it should be a single record.
// If no filter is pushed down to Parquet, it should be the total length of data.
Expand All @@ -2087,15 +2077,7 @@ index 8e88049f51e..49f2001dc6b 100644
}
}
}
}

- test("Filters should be pushed down for Parquet readers at row group level") {
+ test("Filters should be pushed down for Parquet readers at row group level",
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3320")) {
import testImplicits._

withSQLConf(
@@ -1607,7 +1618,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1607,7 +1616,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
// than the total length but should not be a single record.
// Note that, if record level filtering is enabled, it should be a single record.
// If no filter is pushed down to Parquet, it should be the total length of data.
Expand All @@ -2108,7 +2090,7 @@ index 8e88049f51e..49f2001dc6b 100644
}
}
}
@@ -1699,7 +1714,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1699,7 +1712,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
(attr, value) => sources.StringContains(attr, value))
}

Expand All @@ -2117,7 +2099,7 @@ index 8e88049f51e..49f2001dc6b 100644
import testImplicits._
// keep() should take effect on StartsWith/EndsWith/Contains
Seq(
@@ -1743,7 +1758,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1743,7 +1756,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
}
}

Expand All @@ -2127,7 +2109,7 @@ index 8e88049f51e..49f2001dc6b 100644
val schema = StructType(Seq(
StructField("a", IntegerType, nullable = false)
))
@@ -1933,7 +1949,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1933,7 +1947,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
}
}

Expand All @@ -2137,7 +2119,7 @@ index 8e88049f51e..49f2001dc6b 100644
withTempPath { dir =>
val count = 10
val tableName = "spark_25207"
@@ -1984,7 +2001,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1984,7 +1999,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
}
}

Expand All @@ -2147,7 +2129,7 @@ index 8e88049f51e..49f2001dc6b 100644
// block 1:
// null count min max
// page-0 0 0 99
@@ -2044,7 +2062,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -2044,7 +2060,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
}
}

Expand All @@ -2157,7 +2139,7 @@ index 8e88049f51e..49f2001dc6b 100644
withTempPath { dir =>
val path = dir.getCanonicalPath
spark.range(100).selectExpr("id * 2 AS id")
@@ -2276,7 +2295,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
@@ -2276,7 +2293,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
assert(pushedParquetFilters.exists(_.getClass === filterClass),
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")

Expand All @@ -2170,7 +2152,7 @@ index 8e88049f51e..49f2001dc6b 100644
} else {
assert(selectedFilters.isEmpty, "There is filter pushed down")
}
@@ -2336,7 +2359,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
@@ -2336,7 +2357,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
assert(pushedParquetFilters.exists(_.getClass === filterClass),
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")

Expand Down
Loading