Skip to content

[MINOR] Refactor operator-enable configs and remove dead code #12212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
philo-he wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[MINOR] Refactor operator-enable configs and remove dead code #12212

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f3cf8f0
Initial
philo-he Jun 1, 2026
80915bd
Fix comment
philo-he Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,8 +363,6 @@ object CHBackendSettings extends BackendSettingsApi with Logging {

override def structFieldToLowerCase(): Boolean = false

override def supportExpandExec(): Boolean = true

override def excludeScanExecFromCollapsedStage(): Boolean =
SQLConf.get
.getConfString(GLUTEN_CLICKHOUSE_SEP_SCAN_RDD, GLUTEN_CLICKHOUSE_SEP_SCAN_RDD_DEFAULT)
Expand Down
2 changes: 1 addition & 1 deletion ...lickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPl
GlutenConfig.get.enableColumnarBroadcastExchange

private val enableColumnarBroadcastNestedLoopJoin: Boolean =
GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled &&
GlutenConfig.get.enableColumnarBroadcastNestedLoopJoin &&
GlutenConfig.get.enableColumnarBroadcastExchange

override def apply(plan: SparkPlan): SparkPlan = {
Expand Down
2 changes: 0 additions & 2 deletions backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,8 +397,6 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
}

override def supportExpandExec(): Boolean = true

override def supportSortExec(): Boolean = true

override def supportSortMergeJoinExec(): Boolean = {
Expand Down
2 changes: 0 additions & 2 deletions gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ trait BackendSettingsApi {

def supportNativeRowIndexColumn(): Boolean = true

def supportExpandExec(): Boolean = false

def supportSortExec(): Boolean = false

def supportSortMergeJoinExec(): Boolean = true
Expand Down
20 changes: 10 additions & 10 deletions gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,11 +285,11 @@ class GlutenConfig(conf: SQLConf) extends GlutenCoreConfig(conf) {

def fallbackPreferColumnar: Boolean = getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR)

def cartesianProductTransformerEnabled: Boolean =
getConf(CARTESIAN_PRODUCT_TRANSFORMER_ENABLED)
def enableColumnarCartesianProduct: Boolean =
getConf(COLUMNAR_CARTESIAN_PRODUCT_ENABLED)

def broadcastNestedLoopJoinTransformerTransformerEnabled: Boolean =
getConf(BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED)
def enableColumnarBroadcastNestedLoopJoin: Boolean =
getConf(COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED)

def transformPlanLogLevel: String = getConf(TRANSFORM_PLAN_LOG_LEVEL)

Expand Down Expand Up @@ -1462,15 +1462,15 @@ object GlutenConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(true)

val CARTESIAN_PRODUCT_TRANSFORMER_ENABLED =
buildConf("spark.gluten.sql.cartesianProductTransformerEnabled")
.doc("Config to enable CartesianProductExecTransformer.")
val COLUMNAR_CARTESIAN_PRODUCT_ENABLED =
buildConf("spark.gluten.sql.columnar.cartesianProduct")
.doc("Enable or disable columnar cartesianProduct.")
.booleanConf
.createWithDefault(true)

val BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED =
buildConf("spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled")
.doc("Config to enable BroadcastNestedLoopJoinExecTransformer.")
val COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED =
buildConf("spark.gluten.sql.columnar.broadcastNestedLoopJoin")
Copy link
Copy Markdown
Contributor

@marin-ma marin-ma Jun 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should update the docs along with the configuration naming changes

I noticed that the current names are inconsistent for the operator-level enablement configurations. Maybe we can unify the naming using a fixed pattern, such as spark.gluten.sql.columnar.xxx.enabled?

Copy link
Copy Markdown
Member Author

@philo-he philo-he Jun 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marin-ma, thank you so much for the review. I just updated the PR according to your comment. Could you take a look again when you get a chance? Thank you.

.doc("Enable or disable columnar broadcastNestedLoopJoin.")
.booleanConf
.createWithDefault(true)

Expand Down
6 changes: 0 additions & 6 deletions ...t/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.apache.gluten.execution

import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.metrics.MetricsUpdater
import org.apache.gluten.substrait.{JoinParams, SubstraitContext}
import org.apache.gluten.utils.SubstraitUtil
Expand Down Expand Up @@ -167,11 +166,6 @@ abstract class BroadcastNestedLoopJoinExecTransformer(
}

override protected def doValidateInternal(): ValidationResult = {
if (!GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled) {
return ValidationResult.failed(
s"Config ${GlutenConfig.BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED.key} not enabled")
}

if (substraitJoinType == CrossRel.JoinType.UNRECOGNIZED) {
return ValidationResult.failed(
s"$joinType join is not supported with BroadcastNestedLoopJoin")
Expand Down
3 changes: 0 additions & 3 deletions gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,6 @@ case class ExpandExecTransformer(
}

override protected def doValidateInternal(): ValidationResult = {
if (!BackendsApiManager.getSettings.supportExpandExec()) {
return ValidationResult.failed("Current backend does not support expand")
}
if (projections.isEmpty) {
return ValidationResult.failed("Current backend does not support empty projections in expand")
}
Expand Down
4 changes: 2 additions & 2 deletions ...-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ object Validators {
fail(p)
case p: GenerateExec if !glutenConf.enableColumnarGenerate => fail(p)
case p: CoalesceExec if !glutenConf.enableColumnarCoalesce => fail(p)
case p: CartesianProductExec if !glutenConf.cartesianProductTransformerEnabled => fail(p)
case p: CartesianProductExec if !glutenConf.enableColumnarCartesianProduct => fail(p)
case p: TakeOrderedAndProjectExec
if !(glutenConf.enableTakeOrderedAndProject && glutenConf.enableColumnarSort &&
glutenConf.enableColumnarShuffle && glutenConf.enableColumnarProject) =>
Expand All @@ -198,7 +198,7 @@ object Validators {
fail(p)
case p: BroadcastNestedLoopJoinExec
if !(glutenConf.enableColumnarBroadcastJoin &&
glutenConf.broadcastNestedLoopJoinTransformerTransformerEnabled) =>
glutenConf.enableColumnarBroadcastNestedLoopJoin) =>
fail(p)
case p @ (_: HashAggregateExec | _: SortAggregateExec | _: ObjectHashAggregateExec)
if !glutenConf.enableColumnarHashAgg =>
Expand Down
Loading