Add Apache Spark 4.0 support (#787)

.github/workflows/ci.yml

@@ -31,13 +31,19 @@ jobs:
       matrix:
         os: [ubuntu-22.04]
         scala: [2.13, 2.12]
-        java: [temurin@8]
-        project: [root-spark33, root-spark34, root-spark35]
+        java: [temurin@8, temurin@17]
+        project: [root-spark34, root-spark35, root-spark40]
         exclude:
-          - scala: 2.13
-            project: root-spark33
           - scala: 2.13
             project: root-spark34
+          - scala: 2.12
+            project: root-spark40
+          - java: temurin@17
+            project: root-spark34
+          - java: temurin@17
+            project: root-spark35
+          - java: temurin@8
+            project: root-spark40
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     steps:
@@ -62,6 +68,19 @@ jobs:
         if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'
         run: sbt +update
 
+      - name: Setup Java (temurin@17)
+        id: setup-java-temurin-17
+        if: matrix.java == 'temurin@17'
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: sbt
+
+      - name: sbt update
+        if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false'
+        run: sbt +update
+
       - name: Check that workflows are up to date
         run: sbt githubWorkflowCheck
 
@@ -115,6 +134,19 @@ jobs:
         if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'
         run: sbt +update
 
+      - name: Setup Java (temurin@17)
+        id: setup-java-temurin-17
+        if: matrix.java == 'temurin@17'
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: sbt
+
+      - name: sbt update
+        if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false'
+        run: sbt +update
+
       - name: Import signing key
         if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == ''
         env:
@@ -169,18 +201,31 @@ jobs:
         if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'
         run: sbt +update
 
+      - name: Setup Java (temurin@17)
+        id: setup-java-temurin-17
+        if: matrix.java == 'temurin@17'
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: sbt
+
+      - name: sbt update
+        if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false'
+        run: sbt +update
+
       - name: Submit Dependencies
         uses: scalacenter/sbt-dependency-submission@v2
         with:
-          modules-ignore: root-spark33_2.13 root-spark33_2.12 docs_2.13 docs_2.12 root-spark34_2.13 root-spark34_2.12 root-spark35_2.13 root-spark35_2.12
+          modules-ignore: docs_2.13 docs_2.12 root-spark34_2.13 root-spark34_2.12 root-spark35_2.13 root-spark35_2.12 root-spark40_2.13
           configs-ignore: test scala-tool scala-doc-tool test-internal
 
   site:
     name: Generate Site
     strategy:
       matrix:
         os: [ubuntu-22.04]
-        java: [temurin@11]
+        java: [temurin@17]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout current branch (full)
@@ -204,17 +249,17 @@ jobs:
         if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'
         run: sbt +update
 
-      - name: Setup Java (temurin@11)
-        id: setup-java-temurin-11
-        if: matrix.java == 'temurin@11'
+      - name: Setup Java (temurin@17)
+        id: setup-java-temurin-17
+        if: matrix.java == 'temurin@17'
         uses: actions/setup-java@v5
         with:
           distribution: temurin
-          java-version: 11
+          java-version: 17
           cache: sbt
 
       - name: sbt update
-        if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false'
+        if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false'
         run: sbt +update
 
       - name: Generate site

README.md

@@ -27,6 +27,7 @@ The compatible versions of [Spark](http://spark.apache.org/) and
 
 | Frameless | Spark                       | Cats     | Cats-Effect | Scala       |
 |-----------|-----------------------------|----------|-------------|-------------|
+| 0.17.0    | 4.0.2† / 3.5.8 / 3.4.4 / 3.3.4 | 2.x   | 3.x         | 2.12 / 2.13 |
 | 0.16.0    | 3.5.0 / 3.4.0 / 3.3.0       | 2.x      | 3.x         | 2.12 / 2.13 |
 | 0.15.0    | 3.4.0 / 3.3.0 / 3.2.2       | 2.x      | 3.x         | 2.12 / 2.13 |
 | 0.14.1    | 3.4.0 / 3.3.0 / 3.2.2       | 2.x      | 3.x         | 2.12 / 2.13 |
@@ -46,16 +47,19 @@ The compatible versions of [Spark](http://spark.apache.org/) and
 
 _\* 0.11.0 has broken Spark 3.1.2 and 3.0.1 artifacts published._
 
+_† The Spark 4.0.x artifacts (`-spark40`) are published for **Scala 2.13 only** and require **JDK 17+**, since Spark 4 dropped Scala 2.12 and JDK 8/11. The default (unsuffixed) artifacts still target Spark 3.5._
+
 Starting 0.11 we introduced Spark cross published artifacts:
 
 * By default, frameless artifacts depend on the most recent Spark version
 * Suffix `-spark{major}{minor}` is added to artifacts that are released for the previous Spark version(s)
 
 Artifact names examples:
 
-* `frameless-dataset` (the latest Spark dependency)
+* `frameless-dataset` (the default Spark 3.5.x dependency)
+* `frameless-dataset-spark40` (Spark 4.0.x dependency; Scala 2.13 + JDK 17 only)
+* `frameless-dataset-spark34` (Spark 3.4.x dependency)
 * `frameless-dataset-spark33` (Spark 3.3.x dependency)
-* `frameless-dataset-spark32` (Spark 3.2.x dependency)
 
 Versions 0.5.x and 0.6.x have identical features. The first is compatible with Spark 2.2.1 and the second with 2.3.0.