fix: [iceberg] Disable native c2r by default (#3348)

* Revert "chore: Enable native c2r in plan stability suite (#3302)"

This reverts commit 313cf64.

* revert native c2r by default

* fix

* fix

* trigger CI

* Add .gitattributes to collapse golden file diffs on GitHub

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* Revert "Add .gitattributes to collapse golden file diffs on GitHub"

This reverts commit 2fc85d1.

* Fix withInfo test to not depend on CometNativeColumnarToRowExec

Use collectFirst to find CometProjectExec in the plan tree instead of
casting through CometNativeColumnarToRowExec, which is no longer present
when native columnar-to-row is disabled.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* upmerge and fix imports

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: andygrove

common/src/main/scala/org/apache/comet/CometConf.scala

@@ -304,9 +304,9 @@ object CometConf extends ShimCometConf {
         "Whether to enable native columnar to row conversion. When enabled, Comet will use " +
           "native Rust code to convert Arrow columnar data to Spark UnsafeRow format instead " +
           "of the JVM implementation. This can improve performance for queries that need to " +
-          "convert between columnar and row formats.")
+          "convert between columnar and row formats. This is an experimental feature.")
       .booleanConf
-      .createWithDefault(true)
+      .createWithDefault(false)
 
   val COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED: ConfigEntry[Boolean] =
     conf("spark.comet.exec.sortMergeJoinWithJoinFilter.enabled")

spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt

@@ -8,7 +8,7 @@ TakeOrderedAndProject (44)
       :     :  +- * BroadcastHashJoin Inner BuildRight (28)
       :     :     :- * Filter (11)
       :     :     :  +- * HashAggregate (10)
-      :     :     :     +- CometNativeColumnarToRow (9)
+      :     :     :     +- * CometColumnarToRow (9)
       :     :     :        +- CometColumnarExchange (8)
       :     :     :           +- * HashAggregate (7)
       :     :     :              +- * Project (6)
@@ -20,11 +20,11 @@ TakeOrderedAndProject (44)
       :     :     +- BroadcastExchange (27)
       :     :        +- * Filter (26)
       :     :           +- * HashAggregate (25)
-      :     :              +- CometNativeColumnarToRow (24)
+      :     :              +- * CometColumnarToRow (24)
       :     :                 +- CometColumnarExchange (23)
       :     :                    +- * HashAggregate (22)
       :     :                       +- * HashAggregate (21)
-      :     :                          +- CometNativeColumnarToRow (20)
+      :     :                          +- * CometColumnarToRow (20)
       :     :                             +- CometColumnarExchange (19)
       :     :                                +- * HashAggregate (18)
       :     :                                   +- * Project (17)
@@ -34,12 +34,12 @@ TakeOrderedAndProject (44)
       :     :                                         :     +- Scan parquet spark_catalog.default.store_returns (12)
       :     :                                         +- ReusedExchange (15)
       :     +- BroadcastExchange (34)
-      :        +- CometNativeColumnarToRow (33)
+      :        +- * CometColumnarToRow (33)
       :           +- CometProject (32)
       :              +- CometFilter (31)
       :                 +- CometNativeScan parquet spark_catalog.default.store (30)
       +- BroadcastExchange (41)
-         +- CometNativeColumnarToRow (40)
+         +- * CometColumnarToRow (40)
             +- CometProject (39)
                +- CometFilter (38)
                   +- CometNativeScan parquet spark_catalog.default.customer (37)
@@ -53,27 +53,27 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr
 PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)]
 ReadSchema: struct<sr_customer_sk:int,sr_store_sk:int,sr_return_amt:decimal(7,2)>
 
-(2) ColumnarToRow [codegen id : 1]
+(2) ColumnarToRow [codegen id : 2]
 Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
 
-(3) Filter [codegen id : 1]
+(3) Filter [codegen id : 2]
 Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4]
 Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1))
 
 (4) ReusedExchange [Reuses operator id: 49]
 Output [1]: [d_date_sk#6]
 
-(5) BroadcastHashJoin [codegen id : 1]
+(5) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [sr_returned_date_sk#4]
 Right keys [1]: [d_date_sk#6]
 Join type: Inner
 Join condition: None
 
-(6) Project [codegen id : 1]
+(6) Project [codegen id : 2]
 Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3]
 Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#6]
 
-(7) HashAggregate [codegen id : 1]
+(7) HashAggregate [codegen id : 2]
 Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3]
 Keys [2]: [sr_customer_sk#1, sr_store_sk#2]
 Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))]
@@ -84,17 +84,17 @@ Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8]
 Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8]
 Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1]
 
-(9) CometNativeColumnarToRow
+(9) CometColumnarToRow [codegen id : 9]
 Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8]
 
-(10) HashAggregate [codegen id : 5]
+(10) HashAggregate [codegen id : 9]
 Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8]
 Keys [2]: [sr_customer_sk#1, sr_store_sk#2]
 Functions [1]: [sum(UnscaledValue(sr_return_amt#3))]
 Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9]
 Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#10, sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12]
 
-(11) Filter [codegen id : 5]
+(11) Filter [codegen id : 9]
 Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12]
 Condition : isnotnull(ctr_total_return#12)
 
@@ -106,27 +106,27 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#16), dynamicpruningexpression(s
 PushedFilters: [IsNotNull(sr_store_sk)]
 ReadSchema: struct<sr_customer_sk:int,sr_store_sk:int,sr_return_amt:decimal(7,2)>
 
-(13) ColumnarToRow [codegen id : 2]
+(13) ColumnarToRow [codegen id : 4]
 Input [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16]
 
-(14) Filter [codegen id : 2]
+(14) Filter [codegen id : 4]
 Input [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16]
 Condition : isnotnull(sr_store_sk#14)
 
 (15) ReusedExchange [Reuses operator id: 49]
 Output [1]: [d_date_sk#17]
 
-(16) BroadcastHashJoin [codegen id : 2]
+(16) BroadcastHashJoin [codegen id : 4]
 Left keys [1]: [sr_returned_date_sk#16]
 Right keys [1]: [d_date_sk#17]
 Join type: Inner
 Join condition: None
 
-(17) Project [codegen id : 2]
+(17) Project [codegen id : 4]
 Output [3]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15]
 Input [5]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16, d_date_sk#17]
 
-(18) HashAggregate [codegen id : 2]
+(18) HashAggregate [codegen id : 4]
 Input [3]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15]
 Keys [2]: [sr_customer_sk#13, sr_store_sk#14]
 Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#15))]
@@ -137,17 +137,17 @@ Results [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19]
 Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19]
 Arguments: hashpartitioning(sr_customer_sk#13, sr_store_sk#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2]
 
-(20) CometNativeColumnarToRow
+(20) CometColumnarToRow [codegen id : 5]
 Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19]
 
-(21) HashAggregate [codegen id : 3]
+(21) HashAggregate [codegen id : 5]
 Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19]
 Keys [2]: [sr_customer_sk#13, sr_store_sk#14]
 Functions [1]: [sum(UnscaledValue(sr_return_amt#15))]
 Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#15))#9]
 Results [2]: [sr_store_sk#14 AS ctr_store_sk#20, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#9,17,2) AS ctr_total_return#21]
 
-(22) HashAggregate [codegen id : 3]
+(22) HashAggregate [codegen id : 5]
 Input [2]: [ctr_store_sk#20, ctr_total_return#21]
 Keys [1]: [ctr_store_sk#20]
 Functions [1]: [partial_avg(ctr_total_return#21)]
@@ -158,31 +158,31 @@ Results [3]: [ctr_store_sk#20, sum#24, count#25]
 Input [3]: [ctr_store_sk#20, sum#24, count#25]
 Arguments: hashpartitioning(ctr_store_sk#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3]
 
-(24) CometNativeColumnarToRow
+(24) CometColumnarToRow [codegen id : 6]
 Input [3]: [ctr_store_sk#20, sum#24, count#25]
 
-(25) HashAggregate [codegen id : 4]
+(25) HashAggregate [codegen id : 6]
 Input [3]: [ctr_store_sk#20, sum#24, count#25]
 Keys [1]: [ctr_store_sk#20]
 Functions [1]: [avg(ctr_total_return#21)]
 Aggregate Attributes [1]: [avg(ctr_total_return#21)#26]
 Results [2]: [(avg(ctr_total_return#21)#26 * 1.2) AS (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20]
 
-(26) Filter [codegen id : 4]
+(26) Filter [codegen id : 6]
 Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20]
 Condition : isnotnull((avg(ctr_total_return) * 1.2)#27)
 
 (27) BroadcastExchange
 Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20]
 Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4]
 
-(28) BroadcastHashJoin [codegen id : 5]
+(28) BroadcastHashJoin [codegen id : 9]
 Left keys [1]: [ctr_store_sk#11]
 Right keys [1]: [ctr_store_sk#20]
 Join type: Inner
 Join condition: (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27)
 
-(29) Project [codegen id : 5]
+(29) Project [codegen id : 9]
 Output [2]: [ctr_customer_sk#10, ctr_store_sk#11]
 Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20]
 
@@ -201,20 +201,20 @@ Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharC
 Input [2]: [s_store_sk#28, s_state#29]
 Arguments: [s_store_sk#28], [s_store_sk#28]
 
-(33) CometNativeColumnarToRow
+(33) CometColumnarToRow [codegen id : 7]
 Input [1]: [s_store_sk#28]
 
 (34) BroadcastExchange
 Input [1]: [s_store_sk#28]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5]
 
-(35) BroadcastHashJoin [codegen id : 5]
+(35) BroadcastHashJoin [codegen id : 9]
 Left keys [1]: [ctr_store_sk#11]
 Right keys [1]: [s_store_sk#28]
 Join type: Inner
 Join condition: None
 
-(36) Project [codegen id : 5]
+(36) Project [codegen id : 9]
 Output [1]: [ctr_customer_sk#10]
 Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, s_store_sk#28]
 
@@ -233,20 +233,20 @@ Condition : isnotnull(c_customer_sk#30)
 Input [2]: [c_customer_sk#30, c_customer_id#31]
 Arguments: [c_customer_sk#30, c_customer_id#32], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#32]
 
-(40) CometNativeColumnarToRow
+(40) CometColumnarToRow [codegen id : 8]
 Input [2]: [c_customer_sk#30, c_customer_id#32]
 
 (41) BroadcastExchange
 Input [2]: [c_customer_sk#30, c_customer_id#32]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6]
 
-(42) BroadcastHashJoin [codegen id : 5]
+(42) BroadcastHashJoin [codegen id : 9]
 Left keys [1]: [ctr_customer_sk#10]
 Right keys [1]: [c_customer_sk#30]
 Join type: Inner
 Join condition: None
 
-(43) Project [codegen id : 5]
+(43) Project [codegen id : 9]
 Output [1]: [c_customer_id#32]
 Input [3]: [ctr_customer_sk#10, c_customer_sk#30, c_customer_id#32]
 
@@ -258,7 +258,7 @@ Arguments: 100, [c_customer_id#32 ASC NULLS FIRST], [c_customer_id#32]
 
 Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5
 BroadcastExchange (49)
-+- CometNativeColumnarToRow (48)
++- * CometColumnarToRow (48)
    +- CometProject (47)
       +- CometFilter (46)
          +- CometNativeScan parquet spark_catalog.default.date_dim (45)
@@ -279,7 +279,7 @@ Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_
 Input [2]: [d_date_sk#6, d_year#33]
 Arguments: [d_date_sk#6], [d_date_sk#6]
 
-(48) CometNativeColumnarToRow
+(48) CometColumnarToRow [codegen id : 1]
 Input [1]: [d_date_sk#6]
 
 (49) BroadcastExchange

spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/extended.txt

@@ -7,7 +7,7 @@ TakeOrderedAndProject
       :     :  +- BroadcastHashJoin
       :     :     :- Filter
       :     :     :  +- HashAggregate
-      :     :     :     +- CometNativeColumnarToRow
+      :     :     :     +- CometColumnarToRow
       :     :     :        +- CometColumnarExchange
       :     :     :           +- HashAggregate
       :     :     :              +- Project
@@ -17,23 +17,23 @@ TakeOrderedAndProject
       :     :     :                    :     +-  Scan parquet spark_catalog.default.store_returns [COMET: Native DataFusion scan does not support subqueries/dynamic pruning]
       :     :     :                    :           +- SubqueryBroadcast
       :     :     :                    :              +- BroadcastExchange
-      :     :     :                    :                 +- CometNativeColumnarToRow
+      :     :     :                    :                 +- CometColumnarToRow
       :     :     :                    :                    +- CometProject
       :     :     :                    :                       +- CometFilter
       :     :     :                    :                          +- CometNativeScan parquet spark_catalog.default.date_dim
       :     :     :                    +- BroadcastExchange
-      :     :     :                       +- CometNativeColumnarToRow
+      :     :     :                       +- CometColumnarToRow
       :     :     :                          +- CometProject
       :     :     :                             +- CometFilter
       :     :     :                                +- CometNativeScan parquet spark_catalog.default.date_dim
       :     :     +- BroadcastExchange
       :     :        +- Filter
       :     :           +- HashAggregate
-      :     :              +- CometNativeColumnarToRow
+      :     :              +- CometColumnarToRow
       :     :                 +- CometColumnarExchange
       :     :                    +- HashAggregate
       :     :                       +- HashAggregate
-      :     :                          +- CometNativeColumnarToRow
+      :     :                          +- CometColumnarToRow
       :     :                             +- CometColumnarExchange
       :     :                                +- HashAggregate
       :     :                                   +- Project
@@ -43,17 +43,17 @@ TakeOrderedAndProject
       :     :                                         :     +-  Scan parquet spark_catalog.default.store_returns [COMET: Native DataFusion scan does not support subqueries/dynamic pruning]
       :     :                                         :           +- ReusedSubquery
       :     :                                         +- BroadcastExchange
-      :     :                                            +- CometNativeColumnarToRow
+      :     :                                            +- CometColumnarToRow
       :     :                                               +- CometProject
       :     :                                                  +- CometFilter
       :     :                                                     +- CometNativeScan parquet spark_catalog.default.date_dim
       :     +- BroadcastExchange
-      :        +- CometNativeColumnarToRow
+      :        +- CometColumnarToRow
       :           +- CometProject
       :              +- CometFilter
       :                 +- CometNativeScan parquet spark_catalog.default.store
       +- BroadcastExchange
-         +- CometNativeColumnarToRow
+         +- CometColumnarToRow
             +- CometProject
                +- CometFilter
                   +- CometNativeScan parquet spark_catalog.default.customer