Spark解析JSON字符串

bash 复制代码

  {"zhang":{"pid":"alkjdlj","pname":"oweiwuio","page":"werwrw"}}

映射结构 : 第一层MAP,里面切套 MAP,key是String类型，value是string类型

bash 复制代码

val jsonMapper = MapType(StringType, MapType(StringType, StringType))

使用 schema_of_json 动态推断每个键的结构
s_desc 是传入的json格式的字段名称 , json_map是下文解析的变量值

bash 复制代码

    val dynamicResult = frame.withColumn("json_map", from_json(col("s_desc"), jsonMapper))
      // TODO:  person_key 是第一层嵌套的值，必须用这个名称才能取到,explode函数对json_map进行行转列
      .select(explode(col("json_map")).as(Seq("person_key", "person_data")))
      // TODO: 每一列里面进行取值
      .select(
        col("person_key"),
        col("person_data").getItem("pid").as("pid"),
        col("person_data").getItem("pname").as("pname"),
        col("person_data").getItem("page").as("page")
      )

bash 复制代码

package com.zxl

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.dsl.expressions.{DslExpression, StringToAttributeConversionHelper}
import org.apache.spark.sql.functions.{col, explode, from_json, get_json_object, posexplode}
import org.apache.spark.sql.types.{IntegerType, MapType, StringType, StructField, StructType}

import java.util.Properties

object JSONTest {
  def main(args: Array[String]): Unit = {
    // TODO: 创建上下文
    val sparkName = new SparkConf().setMaster("local[*]").setAppName("sparkName")
    // TODO: 创建sparkSession
    val session = SparkSession.builder().config(sparkName).getOrCreate()
    // TODO: 配置JDBC配置，连接mysql
    val props: Properties = new Properties()
    props.setProperty("user", "root")
    props.setProperty("password", "1234")
    // TODO: 配置 school 表连接
    val school = session.read.jdbc("jdbc:mysql://localhost:3306/world", "school", props)
    // TODO: 注册视图表 school
    school.createOrReplaceTempView("school")
    val frame = session.sql("select * from school")

    // TODO:  {"zhang":{"pid":"alkjdlj","pname":"oweiwuio","page":"werwrw"}}
    // TODO: 映射结构 : 第一层MAP,里面切套 MAP,key是String类型，value是string类型
    val jsonMapper = MapType(StringType, MapType(StringType, StringType))
    // TODO:   使用 schema_of_json 动态推断每个键的结构
    // TODO:  s_desc 是传入的json格式的字段名称 , json_map是下文解析的变量值
    val dynamicResult = frame.withColumn("json_map", from_json(col("s_desc"), jsonMapper))
      // TODO:  person_key 是第一层嵌套的值，必须用这个名称才能取到,explode函数对json_map进行行转列
      .select(explode(col("json_map")).as(Seq("person_key", "person_data")))
      // TODO: 每一列里面进行取值
      .select(
        col("person_key"),
        col("person_data").getItem("pid").as("pid"),
        col("person_data").getItem("pname").as("pname"),
        col("person_data").getItem("page").as("page")
      )

    dynamicResult.createOrReplaceTempView("zhangMs")


    // TODO: 配置 person 表连接
    val person = session.read.jdbc("jdbc:mysql://localhost:3306/world", "person", props)
    // TODO: 注册视图表 person
    person.createOrReplaceTempView("person")
    session.sql("select * from zhangMs Left join person on zhangMs.person_key=person.pid").show()



    session.stop()
    session.close()
  }
}