Merge branch 'main' into iceberg-split-serialization-dpp

Author: mbutrovich

.github/actions/java-test/action.yaml

@@ -32,7 +32,7 @@ inputs:
   scan_impl:
     description: 'The default Parquet scan implementation'
     required: false
-    default: 'native_comet'
+    default: 'auto'
   upload-test-reports:
     description: 'Whether to upload test results including coverage to GitHub'
     required: false
@@ -146,7 +146,3 @@ runs:
          path: "**/target/surefire-reports/*.txt"
          retention-days: 7 # 1 week for test reports
          overwrite: true
-
-    - name: Upload coverage results
-      if: ${{ inputs.upload-test-reports == 'true' }}
-      uses: codecov/codecov-action@v5

.github/workflows/docs.yaml

@@ -47,7 +47,7 @@ jobs:
           python-version: "3.10"
 
       - name: Setup Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
           java-version: '17'

.github/workflows/pr_build_linux.yml

@@ -164,7 +164,7 @@ jobs:
           - name: "Spark 3.4, JDK 11, Scala 2.12"
             java_version: "11"
             maven_opts: "-Pspark-3.4 -Pscala-2.12"
-            scan_impl: "native_comet"
+            scan_impl: "auto"
 
           - name: "Spark 3.5.5, JDK 17, Scala 2.13"
             java_version: "17"
@@ -174,7 +174,7 @@ jobs:
           - name: "Spark 3.5.6, JDK 17, Scala 2.13"
             java_version: "17"
             maven_opts: "-Pspark-3.5 -Dspark.version=3.5.6 -Pscala-2.13"
-            scan_impl: "native_comet"
+            scan_impl: "auto"
 
           - name: "Spark 3.5, JDK 17, Scala 2.12"
             java_version: "17"

.github/workflows/spark_sql_test.yml

@@ -116,15 +116,12 @@ jobs:
           - {name: "sql_hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
         # Test combinations:
         # - auto scan: all Spark versions (3.4, 3.5, 4.0)
-        # - native_comet: Spark 3.4, 3.5
         # - native_iceberg_compat: Spark 3.5 only
         config:
           - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'auto', scan-env: ''}
           - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'auto', scan-env: ''}
-          - {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto', scan-env: ''}
-          - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'native_comet', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_comet'}
-          - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'native_comet', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_comet'}
           - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'native_iceberg_compat', scan-env: 'COMET_PARQUET_SCAN_IMPL=native_iceberg_compat'}
+          - {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto', scan-env: ''}
         # Skip sql_hive-1 for Spark 4.0 due to https://github.com/apache/datafusion-comet/issues/2946
         exclude:
           - config: {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto', scan-env: ''}

Makefile

@@ -51,26 +51,26 @@ format:
 
 # build native libs for amd64 architecture Linux/MacOS on a Linux/amd64 machine/container
 core-amd64-libs:
-	cd native && cargo build -j 2 --release $(FEATURES_ARG)
+	cd native && RUSTFLAGS="-Ctarget-cpu=x86-64-v3" cargo build -j 2 --release $(FEATURES_ARG)
 ifdef HAS_OSXCROSS
 	rustup target add x86_64-apple-darwin
 	cd native && cargo build -j 2 --target x86_64-apple-darwin --release $(FEATURES_ARG)
 endif
 
 # build native libs for arm64 architecture Linux/MacOS on a Linux/arm64 machine/container
 core-arm64-libs:
-	cd native && cargo build -j 2 --release $(FEATURES_ARG)
+	cd native && RUSTFLAGS="-Ctarget-cpu=neoverse-n1" cargo build -j 2 --release $(FEATURES_ARG)
 ifdef HAS_OSXCROSS
 	rustup target add aarch64-apple-darwin
 	cd native && cargo build -j 2 --target aarch64-apple-darwin --release $(FEATURES_ARG)
 endif
 
 core-amd64:
 	rustup target add x86_64-apple-darwin
-	cd native && RUSTFLAGS="-Ctarget-cpu=skylake -Ctarget-feature=-prefer-256-bit" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release $(FEATURES_ARG)
+	cd native && RUSTFLAGS="-Ctarget-cpu=skylake" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release $(FEATURES_ARG)
 	mkdir -p common/target/classes/org/apache/comet/darwin/x86_64
 	cp native/target/x86_64-apple-darwin/release/libcomet.dylib common/target/classes/org/apache/comet/darwin/x86_64
-	cd native && RUSTFLAGS="-Ctarget-cpu=haswell -Ctarget-feature=-prefer-256-bit" cargo build --release $(FEATURES_ARG)
+	cd native && RUSTFLAGS="-Ctarget-cpu=x86-64-v3" cargo build --release $(FEATURES_ARG)
 	mkdir -p common/target/classes/org/apache/comet/linux/amd64
 	cp native/target/release/libcomet.so common/target/classes/org/apache/comet/linux/amd64
 	jar -cf common/target/comet-native-x86_64.jar \
@@ -83,7 +83,7 @@ core-arm64:
 	cd native && RUSTFLAGS="-Ctarget-cpu=apple-m1" CC=arm64-apple-darwin21.4-clang CXX=arm64-apple-darwin21.4-clang++ CARGO_FEATURE_NEON=1 cargo build --target aarch64-apple-darwin --release $(FEATURES_ARG)
 	mkdir -p common/target/classes/org/apache/comet/darwin/aarch64
 	cp native/target/aarch64-apple-darwin/release/libcomet.dylib common/target/classes/org/apache/comet/darwin/aarch64
-	cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release $(FEATURES_ARG)
+	cd native && RUSTFLAGS="-Ctarget-cpu=neoverse-n1" cargo build --release $(FEATURES_ARG)
 	mkdir -p common/target/classes/org/apache/comet/linux/aarch64
 	cp native/target/release/libcomet.so common/target/classes/org/apache/comet/linux/aarch64
 	jar -cf common/target/comet-native-aarch64.jar \
@@ -94,8 +94,8 @@ core-arm64:
 release-linux: clean
 	rustup target add aarch64-apple-darwin x86_64-apple-darwin
 	cd native && RUSTFLAGS="-Ctarget-cpu=apple-m1" CC=arm64-apple-darwin21.4-clang CXX=arm64-apple-darwin21.4-clang++ CARGO_FEATURE_NEON=1 cargo build --target aarch64-apple-darwin --release $(FEATURES_ARG)
-	cd native && RUSTFLAGS="-Ctarget-cpu=skylake -Ctarget-feature=-prefer-256-bit" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release $(FEATURES_ARG)
-	cd native && RUSTFLAGS="-Ctarget-cpu=native -Ctarget-feature=-prefer-256-bit" cargo build --release $(FEATURES_ARG)
+	cd native && RUSTFLAGS="-Ctarget-cpu=skylake" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release $(FEATURES_ARG)
+	cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release $(FEATURES_ARG)
 	./mvnw install -Prelease -DskipTests $(PROFILES)
 release:
 	cd native && RUSTFLAGS="$(RUSTFLAGS) -Ctarget-cpu=native" cargo build --release $(FEATURES_ARG)

common/src/main/java/org/apache/arrow/c/AbstractCometSchemaImporter.java

@@ -23,6 +23,8 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.types.pojo.Field;
 
+import org.apache.comet.IcebergApi;
+
 /** This is a simple wrapper around SchemaImporter to make it accessible from Java Arrow. */
 public abstract class AbstractCometSchemaImporter {
   private final BufferAllocator allocator;
@@ -67,6 +69,7 @@ public FieldVector importVector(ArrowArray array, ArrowSchema schema) {
     return vector;
   }
 
+  @IcebergApi
   public void close() {
     provider.close();
   }

common/src/main/java/org/apache/comet/CometSchemaImporter.java

@@ -23,7 +23,9 @@
 import org.apache.arrow.memory.BufferAllocator;
 
 /** This is a simple wrapper around SchemaImporter to make it accessible from Java Arrow. */
+@IcebergApi
 public class CometSchemaImporter extends AbstractCometSchemaImporter {
+  @IcebergApi
   public CometSchemaImporter(BufferAllocator allocator) {
     super(allocator);
   }

common/src/main/java/org/apache/comet/IcebergApi.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.comet;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that the annotated element is part of the public API used by Apache Iceberg.
+ *
+ * <p>This annotation marks classes, methods, constructors, and fields that form the contract
+ * between Comet and Iceberg. Changes to these APIs may break Iceberg's Comet integration, so
+ * contributors should exercise caution and consider backward compatibility when modifying annotated
+ * elements.
+ *
+ * <p>The Iceberg integration uses Comet's native Parquet reader for accelerated vectorized reads.
+ * See the contributor guide documentation for details on how Iceberg uses these APIs.
+ *
+ * @see <a href="https://iceberg.apache.org/">Apache Iceberg</a>
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.FIELD})
+public @interface IcebergApi {}

common/src/main/java/org/apache/comet/parquet/AbstractColumnReader.java

@@ -28,9 +28,11 @@
 import org.apache.spark.sql.types.TimestampNTZType$;
 
 import org.apache.comet.CometConf;
+import org.apache.comet.IcebergApi;
 import org.apache.comet.vector.CometVector;
 
 /** Base class for Comet Parquet column reader implementations. */
+@IcebergApi
 public abstract class AbstractColumnReader implements AutoCloseable {
   protected static final Logger LOG = LoggerFactory.getLogger(AbstractColumnReader.class);
 
@@ -61,7 +63,7 @@ public abstract class AbstractColumnReader implements AutoCloseable {
   protected int batchSize;
 
   /** A pointer to the native implementation of ColumnReader. */
-  protected long nativeHandle;
+  @IcebergApi protected long nativeHandle;
 
   AbstractColumnReader(
       DataType type,
@@ -96,6 +98,7 @@ String getPath() {
   /**
    * Set the batch size of this reader to be 'batchSize'. Also initializes the native column reader.
    */
+  @IcebergApi
   public void setBatchSize(int batchSize) {
     assert nativeHandle == 0
         : "Native column reader shouldn't be initialized before " + "'setBatchSize' is called";
@@ -113,6 +116,7 @@ public void setBatchSize(int batchSize) {
   /** Returns the {@link CometVector} read by this reader. */
   public abstract CometVector currentBatch();
 
+  @IcebergApi
   @Override
   public void close() {
     if (nativeHandle != 0) {

common/src/main/java/org/apache/comet/parquet/BatchReader.java

@@ -65,6 +65,7 @@
 
 import org.apache.comet.CometConf;
 import org.apache.comet.CometSchemaImporter;
+import org.apache.comet.IcebergApi;
 import org.apache.comet.shims.ShimBatchReader;
 import org.apache.comet.shims.ShimFileFormat;
 import org.apache.comet.vector.CometVector;
@@ -87,6 +88,7 @@
  *   }
  * </pre>
  */
+@IcebergApi
 public class BatchReader extends RecordReader<Void, ColumnarBatch> implements Closeable {
   private static final Logger LOG = LoggerFactory.getLogger(FileReader.class);
   protected static final BufferAllocator ALLOCATOR = new RootAllocator();
@@ -186,9 +188,9 @@ public BatchReader(
   }
 
   /**
-   * @deprecated since 0.10.0, will be removed in 0.11.0.
    * @see <a href="https://github.com/apache/datafusion-comet/issues/2079">Comet Issue #2079</a>
    */
+  @IcebergApi
   public BatchReader(AbstractColumnReader[] columnReaders) {
     // Todo: set useDecimal128 and useLazyMaterialization
     int numColumns = columnReaders.length;
@@ -384,17 +386,17 @@ public void init() throws URISyntaxException, IOException {
   }
 
   /**
-   * @deprecated since 0.10.0, will be removed in 0.11.0.
    * @see <a href="https://github.com/apache/datafusion-comet/issues/2079">Comet Issue #2079</a>
    */
+  @IcebergApi
   public void setSparkSchema(StructType schema) {
     this.sparkSchema = schema;
   }
 
   /**
-   * @deprecated since 0.10.0, will be removed in 0.11.0.
    * @see <a href="https://github.com/apache/datafusion-comet/issues/2079">Comet Issue #2079</a>
    */
+  @IcebergApi
   public AbstractColumnReader[] getColumnReaders() {
     return columnReaders;
   }
@@ -498,6 +500,7 @@ public boolean nextBatch() throws IOException {
     return nextBatch(batchSize);
   }
 
+  @IcebergApi
   public boolean nextBatch(int batchSize) {
     long totalDecodeTime = 0, totalLoadTime = 0;
     for (int i = 0; i < columnReaders.length; i++) {
@@ -524,6 +527,7 @@ public boolean nextBatch(int batchSize) {
     return true;
   }
 
+  @IcebergApi
   @Override
   public void close() throws IOException {
     if (columnReaders != null) {