build: Add spark-4.1 profile and shims

Co-authored-by: Andy Grove <agrove@apache.org>

Author: manuzhang

.github/workflows/pr_build_linux.yml

@@ -190,6 +190,11 @@ jobs:
             java_version: "17"
             maven_opts: "-Pspark-4.0"
             scan_impl: "auto"
+
+          - name: "Spark 4.1, JDK 17"
+            java_version: "17"
+            maven_opts: "-Pspark-4.1"
+            scan_impl: "auto"
         suite:
           - name: "fuzz"
             value: |

.github/workflows/spark_sql_test.yml

@@ -115,20 +115,24 @@ jobs:
           - {name: "sql_hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
           - {name: "sql_hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
         # Test combinations:
-        # - auto scan: all Spark versions (3.4, 3.5, 4.0)
+        # - auto scan: all Spark versions (3.4, 3.5, 4.0, 4.1)
         # - native_comet: Spark 3.4, 3.5
         # - native_iceberg_compat: Spark 3.5 only
         config:
           - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'auto', scan-env: ''}
           - {spark-short: '3.5', spark-full: '3.5.7', java: 11, scan-impl: 'auto', scan-env: ''}
           - {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto', scan-env: ''}
+          - {spark-short: '4.1', spark-full: '4.1.1', java: 17, scan-impl: 'auto', scan-env: ''}
           - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'native_comet', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_comet'}
           - {spark-short: '3.5', spark-full: '3.5.7', java: 11, scan-impl: 'native_comet', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_comet'}
           - {spark-short: '3.5', spark-full: '3.5.7', java: 11, scan-impl: 'native_iceberg_compat', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_iceberg_compat'}
-        # Skip sql_hive-1 for Spark 4.0 due to https://github.com/apache/datafusion-comet/issues/2946
+        # Skip sql_hive-1 for Spark 4.0+ due to https://github.com/apache/datafusion-comet/issues/2946
         exclude:
           - config: {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto', scan-env: ''}
             module: {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
+          - config: { spark-short: '4.1', spark-full: '4.1.1', java: 17, scan-impl: 'auto', scan-env: '' }
+            module: { name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest" }
+
       fail-fast: false
     name: spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}/spark-${{ matrix.config.spark-full }}
     runs-on: ${{ matrix.os }}

…g/apache/comet/shims/CometTypeShim.scala …g/apache/comet/shims/CometTypeShim.scalacommon/src/main/spark-4.0/org/apache/comet/shims/CometTypeShim.scala renamed to common/src/main/spark-4.x/org/apache/comet/shims/CometTypeShim.scala common/src/main/spark-4.0/org/apache/comet/shims/CometTypeShim.scala renamed to common/src/main/spark-4.x/org/apache/comet/shims/CometTypeShim.scala

@@ -50,6 +50,7 @@ use only and should not be used in production yet.
 | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
 | ------------- | ------------ | ------------- | ----------------- | --------------------- |
 | 4.0.1         | 17           | 2.13          | Yes               | Yes                   |
+| 4.1.1         | 17           | 2.13          | Yes               | Yes                   |
 
 Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by
 Cloud Service Providers.

…apache/comet/shims/ShimBatchReader.scala …apache/comet/shims/ShimBatchReader.scalacommon/src/main/spark-4.0/org/apache/comet/shims/ShimBatchReader.scala renamed to common/src/main/spark-4.x/org/apache/comet/shims/ShimBatchReader.scala common/src/main/spark-4.0/org/apache/comet/shims/ShimBatchReader.scala renamed to common/src/main/spark-4.x/org/apache/comet/shims/ShimBatchReader.scala

@@ -642,8 +642,29 @@ under the License.
         <parquet.version>1.15.2</parquet.version>
         <semanticdb.version>4.13.6</semanticdb.version>
         <slf4j.version>2.0.16</slf4j.version>
-        <shims.majorVerSrc>spark-4.0</shims.majorVerSrc>
-        <shims.minorVerSrc>not-needed-yet</shims.minorVerSrc>
+        <shims.majorVerSrc>spark-4.x</shims.majorVerSrc>
+        <shims.minorVerSrc>spark-4.0</shims.minorVerSrc>
+        <!-- Use jdk17 by default -->
+        <java.version>17</java.version>
+        <maven.compiler.source>${java.version}</maven.compiler.source>
+        <maven.compiler.target>${java.version}</maven.compiler.target>
+      </properties>
+    </profile>
+
+    <profile>
+      <!-- FIXME: this is WIP. Tests may fail https://github.com/apache/datafusion-comet/issues/551 -->
+      <id>spark-4.1</id>
+      <properties>
+        <!-- Use Scala 2.13 by default -->
+        <scala.version>2.13.17</scala.version>
+        <scala.binary.version>2.13</scala.binary.version>
+        <spark.version>4.1.1</spark.version>
+        <spark.version.short>4.1</spark.version.short>
+        <parquet.version>1.16.0</parquet.version>
+        <semanticdb.version>4.13.9</semanticdb.version>
+        <slf4j.version>2.0.17</slf4j.version>
+        <shims.majorVerSrc>spark-4.x</shims.majorVerSrc>
+        <shims.minorVerSrc>spark-4.1</shims.minorVerSrc>
         <!-- Use jdk17 by default -->
         <java.version>17</java.version>
         <maven.compiler.source>${java.version}</maven.compiler.source>