Skip to content

Commit 6d9d3d8

Browse files
compheadcomphead
committed
chore: Add native writer Spark tests CI pipeline
1 parent 5c1f131 commit 6d9d3d8

1 file changed

Lines changed: 96 additions & 0 deletions

File tree

‎.github/workflows/spark_sql_test_native_writer.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Spark SQL Tests (native_writer)
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
# manual trigger
26+
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
27+
workflow_dispatch:
28+
29+
env:
30+
RUST_VERSION: stable
31+
32+
jobs:
33+
spark-sql-native-writer:
34+
strategy:
35+
matrix:
36+
os: [ubuntu-24.04]
37+
java-version: [11]
38+
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
39+
module:
40+
# Core DataFrame Writer Test Suites
41+
- {name: "sql/DataFrameReaderWriterSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.test.DataFrameReaderWriterSuite"}
42+
- {name: "sql/DataFrameWriterV2Suite", args1: "", args2: "sql/testOnly org.apache.spark.sql.DataFrameWriterV2Suite"}
43+
- {name: "sql/DataFrameSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.DataFrameSuite"}
44+
- {name: "sql/DataFrameCallbackSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.util.DataFrameCallbackSuite"}
45+
# Java Test Suites
46+
- {name: "sql/JavaDataFrameReaderWriterSuite", args1: "", args2: "sql/testOnly test.org.apache.spark.sql.JavaDataFrameReaderWriterSuite"}
47+
- {name: "sql/JavaDataFrameWriterV2Suite", args1: "", args2: "sql/testOnly test.org.apache.spark.sql.JavaDataFrameWriterV2Suite"}
48+
# Format-Specific Writer Test Suites
49+
- {name: "sql/FileFormatWriterSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.execution.datasources.FileFormatWriterSuite"}
50+
- {name: "sql/FileWriterFactorySuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.execution.datasources.v2.FileWriterFactorySuite"}
51+
- {name: "sql/JDBCWriteSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.jdbc.JDBCWriteSuite"}
52+
# Partitioning and Bucketing Writer Test Suites
53+
- {name: "sql/PartitionedWriteSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.sources.PartitionedWriteSuite"}
54+
- {name: "sql/BucketedWriteSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.sources.BucketedWriteSuite"}
55+
- {name: "sql/BucketedWriteWithoutHiveSupportSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.sources.BucketedWriteWithoutHiveSupportSuite"}
56+
# V1/V2 Write Command Test Suites
57+
- {name: "sql/V1WriteCommandSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.execution.datasources.V1WriteCommandSuite"}
58+
- {name: "sql/V1WriteFallbackSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.connector.V1WriteFallbackSuite"}
59+
- {name: "sql/SaveIntoDataSourceCommandSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommandSuite"}
60+
# Insert and Save/Load Test Suites
61+
- {name: "sql/InsertSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.sources.InsertSuite"}
62+
- {name: "sql/SaveLoadSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.sources.SaveLoadSuite"}
63+
- {name: "sql/CreateTableAsSelectSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.sources.CreateTableAsSelectSuite"}
64+
# Streaming Writer Test Suites
65+
- {name: "sql/DataStreamReaderWriterSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.streaming.test.DataStreamReaderWriterSuite"}
66+
- {name: "sql/FileStreamSinkV1Suite", args1: "", args2: "sql/testOnly org.apache.spark.sql.streaming.FileStreamSinkV1Suite"}
67+
- {name: "sql/FileStreamSinkV2Suite", args1: "", args2: "sql/testOnly org.apache.spark.sql.streaming.FileStreamSinkV2Suite"}
68+
- {name: "sql/ForeachWriterSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.execution.streaming.sources.ForeachWriterSuite"}
69+
- {name: "sql/PythonForeachWriterSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.execution.python.PythonForeachWriterSuite"}
70+
# Other Writer Test Suites
71+
- {name: "sql/ArrowWriterSuite", args1: "", args2: "sql/testOnly org.apache.spark.sql.execution.arrow.ArrowWriterSuite"}
72+
fail-fast: false
73+
name: spark-sql-native-writer-${{ matrix.module.name }}/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
74+
runs-on: ${{ matrix.os }}
75+
container:
76+
image: amd64/rust
77+
steps:
78+
- uses: actions/checkout@v6
79+
- name: Setup Rust & Java toolchain
80+
uses: ./.github/actions/setup-builder
81+
with:
82+
rust-version: ${{env.RUST_VERSION}}
83+
jdk-version: ${{ matrix.java-version }}
84+
- name: Setup Spark
85+
uses: ./.github/actions/setup-spark-builder
86+
with:
87+
spark-version: ${{ matrix.spark-version.full }}
88+
spark-short-version: ${{ matrix.spark-version.short }}
89+
- name: Run Spark tests
90+
run: |
91+
cd apache-spark
92+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
93+
ENABLE_COMET=true ENABLE_COMET_ONHEAP=true COMET_PARQUET_SCAN_IMPL=native_iceberg_compat ENABLE_COMET_WRITE=true COMET_OPERATOR_DATA_WRITING_COMMAND_ALLOW_INCOMPAT=true build/sbt -Dsbt.log.noformat=true ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}"
94+
env:
95+
LC_ALL: "C.UTF-8"
96+

0 commit comments

Comments
 (0)