Spark 之 SparkSessionExtensions

zhixingheyi_tian2024-11-24 17:36

SparkSessionExtensions

org.apache.spark.sql.internal.BaseSessionStateBuilder

复制代码

  /**
   * Session extensions defined in the [[SparkSession]].
   */
  protected def extensions: SparkSessionExtensions = session.extensions

  /**
   * Custom strategies to add to the planner. Prefer overriding this instead of creating
   * your own Planner.
   *
   * Note that this may NOT depend on the `planner` function.
   */
  protected def customPlanningStrategies: Seq[Strategy] = {
    extensions.buildPlannerStrategies(session)
  }

  protected def columnarRules: Seq[ColumnarRule] = {
    extensions.buildColumnarRules(session)
  }

  protected def adaptiveRulesHolder: AdaptiveRulesHolder = {
    new AdaptiveRulesHolder(
      extensions.buildQueryStagePrepRules(session),
      extensions.buildRuntimeOptimizerRules(session),
      extensions.buildQueryStageOptimizerRules(session),
      extensions.buildQueryPostPlannerStrategyRules(session))
  }

  protected def planNormalizationRules: Seq[Rule[LogicalPlan]] = {
    extensions.buildPlanNormalizationRules(session)
  }

extensions 使用的地方

org.apache.spark.sql.execution.QueryExecution.scala

复制代码

      ApplyColumnarRulesAndInsertTransitions(
        sparkSession.sessionState.columnarRules, outputsColumnar = false),