Spark 之 HiveStrategies

HiveTableRelation 相关代码

HiveStrategies.scala

当 relation.tableMeta.stats.isEmpty 是, 即调用 hiveTableWithStats

复制代码
class DetermineTableStats(session: SparkSession) extends Rule[LogicalPlan] {
  private def hiveTableWithStats(relation: HiveTableRelation): HiveTableRelation = {
    val table = relation.tableMeta
    val partitionCols = relation.partitionCols
    // For partitioned tables, the partition directory may be outside of the table directory.
    // Which is expensive to get table size. Please see how we implemented it in the AnalyzeTable.
    val sizeInBytes = if (conf.fallBackToHdfsForStatsEnabled && partitionCols.isEmpty) {
      try {
        val hadoopConf = session.sessionState.newHadoopConf()
        val tablePath = new Path(table.location)
        val fs: FileSystem = tablePath.getFileSystem(hadoopConf)
        fs.getContentSummary(tablePath).getLength
      } catch {
        case e: IOException =>
          logWarning("Failed to get table size from HDFS.", e)
          conf.defaultSizeInBytes
      }
    } else {
      conf.defaultSizeInBytes
    }

    val stats = Some(Statistics(sizeInBytes = BigInt(sizeInBytes)))
    relation.copy(tableStats = stats)
  }

  override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
    case relation: HiveTableRelation
      if DDLUtils.isHiveTable(relation.tableMeta) && relation.tableMeta.stats.isEmpty =>
      hiveTableWithStats(relation)

    // handles InsertIntoStatement specially as the table in InsertIntoStatement is not added in its
    // children, hence not matched directly by previous HiveTableRelation case.
    case i @ InsertIntoStatement(relation: HiveTableRelation, _, _, _, _, _)
      if DDLUtils.isHiveTable(relation.tableMeta) && relation.tableMeta.stats.isEmpty =>
      i.copy(table = hiveTableWithStats(relation))
  }
}
  • HiveTableRelation

    /**

    • A LogicalPlan that represents a hive table.
    • TODO: remove this after we completely make hive as a data source.
      */
      case class HiveTableRelation(
      tableMeta: CatalogTable,
      dataCols: Seq[AttributeReference],
      partitionCols: Seq[AttributeReference],
      tableStats: Option[Statistics] = None,
      @transient prunedPartitions: Option[Seq[CatalogTablePartition]] = None)
相关推荐
注塑机数据采集联网14 分钟前
注塑机智能数据采集联网网关 ZK-D1/D3 如何读取力劲注塑机控制器数据并对接 MES/IOT 平台
大数据·网络·物联网·数据挖掘·注塑机网关
bksczm16 小时前
Redis Cluster 分片集群详解:槽路由、故障判定与扩缩容
数据库·redis·分布式
txxzjmzlh17 小时前
RabbitMQ安装
分布式·rabbitmq
黑十17 小时前
FDE 怎么快速搭一个可演示页面?最小前端方案
java·大数据·前端·数据库·人工智能
Databend18 小时前
OpenRaft 自定义 Membership,按部署拓扑选择 Quorum
大数据·数据库·数据分析
Databend18 小时前
用 SQL 排查支付故障,OpenTelemetry 接入 Databend Cloud 实战
大数据·数据库·sql
上海潭龙东海科技有限公司19 小时前
智慧堆场动态模型三维实时孪生重构智能管控
大数据·网络·数据库·算法·重构
尹文祥20 小时前
Apache Doris 2.1.10:存储结构、索引与查询优化详解
大数据
isfox1 天前
Hadoop 序列化:为什么不用 Java 自带 Serializable?
大数据
字节跳动数据平台1 天前
火山引擎 VeKnow 落地指南(下):从知识编译到价值渗透
大数据