chore: Bump Spark from 3.5.7 to 3.5.8 (#3323)

Author: manuzhang

.github/actions/setup-spark-builder/action.yaml

@@ -22,7 +22,7 @@ inputs:
     description: 'The Apache Spark short version (e.g., 3.5) to build'
     required: true
   spark-version:
-    description: 'The Apache Spark version (e.g., 3.5.7) to build'
+    description: 'The Apache Spark version (e.g., 3.5.8) to build'
     required: true
   skip-native-build:
     description: 'Skip native build (when using pre-built artifact)'

.github/workflows/iceberg_spark_test.yml

@@ -53,7 +53,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
-        spark-version: [{short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.5', full: '3.5.8'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -92,7 +92,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
-        spark-version: [{short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.5', full: '3.5.8'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-extensions/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -131,7 +131,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
-        spark-version: [{short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.5', full: '3.5.8'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-runtime/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -170,7 +170,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-rust/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -209,7 +209,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-extensions-rust/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -248,7 +248,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-runtime-rust/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}

.github/workflows/spark_sql_test.yml

@@ -120,11 +120,11 @@ jobs:
         # - native_iceberg_compat: Spark 3.5 only
         config:
           - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'auto', scan-env: ''}
-          - {spark-short: '3.5', spark-full: '3.5.7', java: 11, scan-impl: 'auto', scan-env: ''}
+          - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'auto', scan-env: ''}
           - {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto', scan-env: ''}
           - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'native_comet', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_comet'}
-          - {spark-short: '3.5', spark-full: '3.5.7', java: 11, scan-impl: 'native_comet', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_comet'}
-          - {spark-short: '3.5', spark-full: '3.5.7', java: 11, scan-impl: 'native_iceberg_compat', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_iceberg_compat'}
+          - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'native_comet', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_comet'}
+          - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'native_iceberg_compat', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_iceberg_compat'}
         # Skip sql_hive-1 for Spark 4.0 due to https://github.com/apache/datafusion-comet/issues/2946
         exclude:
           - config: {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto', scan-env: ''}

.github/workflows/spark_sql_test_native_datafusion.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

.github/workflows/spark_sql_test_native_iceberg_compat.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

dev/diffs/3.5.7.diff dev/diffs/3.5.8.diffdev/diffs/3.5.7.diff renamed to dev/diffs/3.5.8.diff

@@ -41,6 +41,7 @@ Other versions may work well enough for development and evaluation purposes.
 | 3.5.5         | 11/17        | 2.12/2.13     | Yes               | No                    |
 | 3.5.6         | 11/17        | 2.12/2.13     | Yes               | No                    |
 | 3.5.7         | 11/17        | 2.12/2.13     | Yes               | Yes                   |
+| 3.5.8         | 11/17        | 2.12/2.13     | Yes               | Yes                   |
 
 Note that we do not test the full matrix of supported Java and Scala versions in CI for every Spark version.
 

docs/source/user-guide/latest/installation.md

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-FROM apache/spark:3.5.7 AS builder
+FROM apache/spark:3.5.8 AS builder
 
 USER root
 
@@ -70,7 +70,7 @@ RUN mkdir -p /root/.m2 && \
 RUN cd /comet \
     && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
 
-FROM apache/spark:3.5.7
+FROM apache/spark:3.5.8
 ENV SPARK_VERSION=3.5
 ENV SCALA_VERSION=2.12
 USER root

kube/Dockerfile

@@ -70,7 +70,7 @@ under the License.
     <scala.plugin.version>4.9.6</scala.plugin.version>
     <scalatest.version>3.2.16</scalatest.version>
     <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
-    <spark.version>3.5.7</spark.version>
+    <spark.version>3.5.8</spark.version>
     <spark.version.short>3.5</spark.version.short>
     <spark.maven.scope>provided</spark.maven.scope>
     <protobuf.version>3.25.5</protobuf.version>
@@ -619,7 +619,7 @@ under the License.
       <id>spark-3.5</id>
       <properties>
         <scala.version>2.12.18</scala.version>
-        <spark.version>3.5.7</spark.version>
+        <spark.version>3.5.8</spark.version>
         <spark.version.short>3.5</spark.version.short>
         <parquet.version>1.13.1</parquet.version>
         <slf4j.version>2.0.7</slf4j.version>