Spark 之 HiveStrategies

HiveTableRelation 相关代码

HiveStrategies.scala

当 relation.tableMeta.stats.isEmpty 是, 即调用 hiveTableWithStats

复制代码
class DetermineTableStats(session: SparkSession) extends Rule[LogicalPlan] {
  private def hiveTableWithStats(relation: HiveTableRelation): HiveTableRelation = {
    val table = relation.tableMeta
    val partitionCols = relation.partitionCols
    // For partitioned tables, the partition directory may be outside of the table directory.
    // Which is expensive to get table size. Please see how we implemented it in the AnalyzeTable.
    val sizeInBytes = if (conf.fallBackToHdfsForStatsEnabled && partitionCols.isEmpty) {
      try {
        val hadoopConf = session.sessionState.newHadoopConf()
        val tablePath = new Path(table.location)
        val fs: FileSystem = tablePath.getFileSystem(hadoopConf)
        fs.getContentSummary(tablePath).getLength
      } catch {
        case e: IOException =>
          logWarning("Failed to get table size from HDFS.", e)
          conf.defaultSizeInBytes
      }
    } else {
      conf.defaultSizeInBytes
    }

    val stats = Some(Statistics(sizeInBytes = BigInt(sizeInBytes)))
    relation.copy(tableStats = stats)
  }

  override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
    case relation: HiveTableRelation
      if DDLUtils.isHiveTable(relation.tableMeta) && relation.tableMeta.stats.isEmpty =>
      hiveTableWithStats(relation)

    // handles InsertIntoStatement specially as the table in InsertIntoStatement is not added in its
    // children, hence not matched directly by previous HiveTableRelation case.
    case i @ InsertIntoStatement(relation: HiveTableRelation, _, _, _, _, _)
      if DDLUtils.isHiveTable(relation.tableMeta) && relation.tableMeta.stats.isEmpty =>
      i.copy(table = hiveTableWithStats(relation))
  }
}
  • HiveTableRelation

    /**

    • A LogicalPlan that represents a hive table.
    • TODO: remove this after we completely make hive as a data source.
      */
      case class HiveTableRelation(
      tableMeta: CatalogTable,
      dataCols: Seq[AttributeReference],
      partitionCols: Seq[AttributeReference],
      tableStats: Option[Statistics] = None,
      @transient prunedPartitions: Option[Seq[CatalogTablePartition]] = None)
相关推荐
桑榆080621 分钟前
Kafka简介
spark·kafka
24k小善30 分钟前
Flink TaskManager详解
java·大数据·flink·云计算
时序数据说1 小时前
时序数据库IoTDB在航空航天领域的解决方案
大数据·数据库·时序数据库·iotdb
IT成长日记2 小时前
【Hive入门】Hive概述:大数据时代的数据仓库桥梁
大数据·数据仓库·hive·sql优化·分布式计算
科技小E2 小时前
EasyRTC音视频实时通话嵌入式SDK,打造社交娱乐低延迟实时互动的新体验
大数据·网络
降世神童3 小时前
大数据系列 | 详解基于Zookeeper或ClickHouse Keeper的ClickHouse集群部署--完结
大数据·clickhouse·zookeeper
躺不平的理查德3 小时前
General Spark Operations(Spark 基础操作)
大数据·分布式·spark
talle20213 小时前
Zeppelin在spark环境导出dataframe
大数据·分布式·spark
大数据魔法师3 小时前
Hadoop生态圈框架部署 - Windows上部署Hadoop
大数据·hadoop·windows
渣渣盟4 小时前
大数据开发环境的安装,配置(Hadoop)
大数据·hadoop·分布式