diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index ba540914fbc..395ed6fd17f 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -363,8 +363,6 @@ object CHBackendSettings extends BackendSettingsApi with Logging { override def structFieldToLowerCase(): Boolean = false - override def supportExpandExec(): Boolean = true - override def excludeScanExecFromCollapsedStage(): Boolean = SQLConf.get .getConfString(GLUTEN_CLICKHOUSE_SEP_SCAN_RDD, GLUTEN_CLICKHOUSE_SEP_SCAN_RDD_DEFAULT) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala index 2b728a94647..4a9513b4296 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala @@ -151,7 +151,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPl GlutenConfig.get.enableColumnarBroadcastExchange private val enableColumnarBroadcastNestedLoopJoin: Boolean = - GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled && + GlutenConfig.get.enableColumnarBroadcastNestedLoopJoin && GlutenConfig.get.enableColumnarBroadcastExchange override def apply(plan: SparkPlan): SparkPlan = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 599b91851ea..6ecbb745b27 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -397,8 +397,6 @@ object VeloxBackendSettings extends BackendSettingsApi { } } - override def supportExpandExec(): Boolean = true - override def supportSortExec(): Boolean = true override def supportSortMergeJoinExec(): Boolean = { diff --git a/docs/Configuration.md b/docs/Configuration.md index 748a200cfbf..da173c7aaaf 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -41,15 +41,15 @@ nav_order: 15 | spark.gluten.soft-affinity.replications.num | 🔄 Dynamic | 2 | Calculate the number of the replications for scheduling to the target executors per file | | spark.gluten.sql.adaptive.costEvaluator.enabled | ⚓ Static | true | If true, use org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as custom cost evaluator class, else follow the configuration spark.sql.adaptive.customCostEvaluatorClass. | | spark.gluten.sql.ansiFallback.enabled | 🔄 Dynamic | true | When true (default), Gluten will fall back to Spark when ANSI mode is enabled. When false, Gluten will attempt to execute in ANSI mode. | -| spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled | 🔄 Dynamic | true | Config to enable BroadcastNestedLoopJoinExecTransformer. | | spark.gluten.sql.cacheWholeStageTransformerContext | 🔄 Dynamic | false | When true, `WholeStageTransformer` will cache the `WholeStageTransformerContext` when executing. It is used to get substrait plan node and native plan string. | -| spark.gluten.sql.cartesianProductTransformerEnabled | 🔄 Dynamic | true | Config to enable CartesianProductExecTransformer. | | spark.gluten.sql.collapseGetJsonObject.enabled | 🔄 Dynamic | false | Collapse nested get_json_object functions as one for optimization. | | spark.gluten.sql.columnar.appendData | 🔄 Dynamic | true | Enable or disable columnar v2 command append data. | | spark.gluten.sql.columnar.arrowUdf | 🔄 Dynamic | true | Enable or disable columnar arrow udf. | | spark.gluten.sql.columnar.batchscan | 🔄 Dynamic | true | Enable or disable columnar batchscan. | | spark.gluten.sql.columnar.broadcastExchange | 🔄 Dynamic | true | Enable or disable columnar broadcastExchange. | | spark.gluten.sql.columnar.broadcastJoin | 🔄 Dynamic | true | Enable or disable columnar broadcastJoin. | +| spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled | 🔄 Dynamic | true | Enable or disable columnar broadcastNestedLoopJoin. | +| spark.gluten.sql.columnar.cartesianProduct.enabled | 🔄 Dynamic | true | Enable or disable columnar cartesianProduct. | | spark.gluten.sql.columnar.cast.avg | 🔄 Dynamic | true | | spark.gluten.sql.columnar.coalesce | 🔄 Dynamic | true | Enable or disable columnar coalesce. | | spark.gluten.sql.columnar.collectLimit | 🔄 Dynamic | true | Enable or disable columnar collectLimit. | diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index 7f5d5a9c214..980dec57db6 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -63,8 +63,6 @@ trait BackendSettingsApi { def supportNativeRowIndexColumn(): Boolean = true - def supportExpandExec(): Boolean = false - def supportSortExec(): Boolean = false def supportSortMergeJoinExec(): Boolean = true diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala index 821d7472287..6c777aed901 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala @@ -285,11 +285,11 @@ class GlutenConfig(conf: SQLConf) extends GlutenCoreConfig(conf) { def fallbackPreferColumnar: Boolean = getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR) - def cartesianProductTransformerEnabled: Boolean = - getConf(CARTESIAN_PRODUCT_TRANSFORMER_ENABLED) + def enableColumnarCartesianProduct: Boolean = + getConf(COLUMNAR_CARTESIAN_PRODUCT_ENABLED) - def broadcastNestedLoopJoinTransformerTransformerEnabled: Boolean = - getConf(BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED) + def enableColumnarBroadcastNestedLoopJoin: Boolean = + getConf(COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED) def transformPlanLogLevel: String = getConf(TRANSFORM_PLAN_LOG_LEVEL) @@ -1462,15 +1462,15 @@ object GlutenConfig extends ConfigRegistry { .booleanConf .createWithDefault(true) - val CARTESIAN_PRODUCT_TRANSFORMER_ENABLED = - buildConf("spark.gluten.sql.cartesianProductTransformerEnabled") - .doc("Config to enable CartesianProductExecTransformer.") + val COLUMNAR_CARTESIAN_PRODUCT_ENABLED = + buildConf("spark.gluten.sql.columnar.cartesianProduct.enabled") + .doc("Enable or disable columnar cartesianProduct.") .booleanConf .createWithDefault(true) - val BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED = - buildConf("spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled") - .doc("Config to enable BroadcastNestedLoopJoinExecTransformer.") + val COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED = + buildConf("spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled") + .doc("Enable or disable columnar broadcastNestedLoopJoin.") .booleanConf .createWithDefault(true) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala index 282cf8581e1..32a5793678b 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala @@ -17,7 +17,6 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.{JoinParams, SubstraitContext} import org.apache.gluten.utils.SubstraitUtil @@ -167,11 +166,6 @@ abstract class BroadcastNestedLoopJoinExecTransformer( } override protected def doValidateInternal(): ValidationResult = { - if (!GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled) { - return ValidationResult.failed( - s"Config ${GlutenConfig.BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED.key} not enabled") - } - if (substraitJoinType == CrossRel.JoinType.UNRECOGNIZED) { return ValidationResult.failed( s"$joinType join is not supported with BroadcastNestedLoopJoin") diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala index 47aa66f16b2..7cf49d97af7 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala @@ -97,9 +97,6 @@ case class ExpandExecTransformer( } override protected def doValidateInternal(): ValidationResult = { - if (!BackendsApiManager.getSettings.supportExpandExec()) { - return ValidationResult.failed("Current backend does not support expand") - } if (projections.isEmpty) { return ValidationResult.failed("Current backend does not support empty projections in expand") } diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala index faa0d9e2e54..4f056969c4a 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala @@ -189,7 +189,7 @@ object Validators { fail(p) case p: GenerateExec if !glutenConf.enableColumnarGenerate => fail(p) case p: CoalesceExec if !glutenConf.enableColumnarCoalesce => fail(p) - case p: CartesianProductExec if !glutenConf.cartesianProductTransformerEnabled => fail(p) + case p: CartesianProductExec if !glutenConf.enableColumnarCartesianProduct => fail(p) case p: TakeOrderedAndProjectExec if !(glutenConf.enableTakeOrderedAndProject && glutenConf.enableColumnarSort && glutenConf.enableColumnarShuffle && glutenConf.enableColumnarProject) => @@ -198,7 +198,7 @@ object Validators { fail(p) case p: BroadcastNestedLoopJoinExec if !(glutenConf.enableColumnarBroadcastJoin && - glutenConf.broadcastNestedLoopJoinTransformerTransformerEnabled) => + glutenConf.enableColumnarBroadcastNestedLoopJoin) => fail(p) case p @ (_: HashAggregateExec | _: SortAggregateExec | _: ObjectHashAggregateExec) if !glutenConf.enableColumnarHashAgg =>