Spark 之 HiveStrategies

HiveTableRelation 相关代码

HiveStrategies.scala

当 relation.tableMeta.stats.isEmpty 是, 即调用 hiveTableWithStats

复制代码
class DetermineTableStats(session: SparkSession) extends Rule[LogicalPlan] {
  private def hiveTableWithStats(relation: HiveTableRelation): HiveTableRelation = {
    val table = relation.tableMeta
    val partitionCols = relation.partitionCols
    // For partitioned tables, the partition directory may be outside of the table directory.
    // Which is expensive to get table size. Please see how we implemented it in the AnalyzeTable.
    val sizeInBytes = if (conf.fallBackToHdfsForStatsEnabled && partitionCols.isEmpty) {
      try {
        val hadoopConf = session.sessionState.newHadoopConf()
        val tablePath = new Path(table.location)
        val fs: FileSystem = tablePath.getFileSystem(hadoopConf)
        fs.getContentSummary(tablePath).getLength
      } catch {
        case e: IOException =>
          logWarning("Failed to get table size from HDFS.", e)
          conf.defaultSizeInBytes
      }
    } else {
      conf.defaultSizeInBytes
    }

    val stats = Some(Statistics(sizeInBytes = BigInt(sizeInBytes)))
    relation.copy(tableStats = stats)
  }

  override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
    case relation: HiveTableRelation
      if DDLUtils.isHiveTable(relation.tableMeta) && relation.tableMeta.stats.isEmpty =>
      hiveTableWithStats(relation)

    // handles InsertIntoStatement specially as the table in InsertIntoStatement is not added in its
    // children, hence not matched directly by previous HiveTableRelation case.
    case i @ InsertIntoStatement(relation: HiveTableRelation, _, _, _, _, _)
      if DDLUtils.isHiveTable(relation.tableMeta) && relation.tableMeta.stats.isEmpty =>
      i.copy(table = hiveTableWithStats(relation))
  }
}
  • HiveTableRelation

    /**

    • A LogicalPlan that represents a hive table.
    • TODO: remove this after we completely make hive as a data source.
      */
      case class HiveTableRelation(
      tableMeta: CatalogTable,
      dataCols: Seq[AttributeReference],
      partitionCols: Seq[AttributeReference],
      tableStats: Option[Statistics] = None,
      @transient prunedPartitions: Option[Seq[CatalogTablePartition]] = None)
相关推荐
IvanCodes24 分钟前
Oracle 视图
大数据·数据库·sql·oracle
cal_1 小时前
数据分析中的拉链表解析
大数据·数据挖掘·数据分析
亿信华辰软件1 小时前
亿信华辰赋能大学教务数据分析实战记
大数据·数据分析
❀always❀2 小时前
深入浅出分布式限流(更新中)
分布式·wpf
qyresearch_2 小时前
全球机械工业设计服务市场:技术驱动下的创新与增长
大数据·人工智能
前端开发与ui设计的老司机3 小时前
大数据时代UI前端的智能化升级:基于机器学习的用户意图预测
大数据·ui
oneslide3 小时前
elasticsearch备份恢复
大数据·elasticsearch·搜索引擎
rui锐rui4 小时前
大数据学习6:Sqoop数据迁移工具
大数据·学习·sqoop
Bug退退退1234 小时前
RabbitMQ 幂等性
分布式·rabbitmq