pyspark 处理字符串函数

pyspark 要处理数据，没有，那就伪造数据 faker 真是个好东西

复制代码

from faker import Faker
import pandas as pd
gender = ["None","Man","Woman"]

fake = Faker()
names = [(fake.first_name(),fake.last_name(),fake.date_of_birth(),fake.random_int(1,99),gender[fake.random_int(1,2)]) for _ in range(1000)]

pd.DataFrame(names, columns=["first_name","last_name","birthday","age","gender"]).to_csv("fake_names.csv", index=False)

from pyspark.sql import SparkSession
#字符串处理函数
spark = SparkSession.builder.appName('example').getOrCreate()
df =spark.read.csv("fake_names.csv",header=True,inferSchema=True)
# df.show(3)

1、拼接字符串 concat() concat_ws() format_string()

复制代码

from pyspark.sql.functions import concat,concat_ws,format_string
 df.select("first_name","last_name","age",
          concat(df["first_name"],df['last_name']).alias("concat_result"),
          concat_ws('#',df["first_name"],df['last_name']).alias("concat_ws_result"),
         format_string("姓%s名%s年龄%d",df["first_name"],df['last_name'],df['age']).alias("format_string_result")
          ).show(5)

2、字符串的长度和转换大小写 length() lower() upper()

复制代码

from pyspark.sql.functions import length,lower,upper

 df.select(
     "first_name",
     length(df["first_name"]).alias("length_result"),
     lower(df["first_name"]).alias("lower_result"),
     upper(df["first_name"]).alias("upper_result")
).show(5)

3、添加常量值到新的列中

复制代码

from pyspark.sql.functions import lit

df.withColumn("身份",lit('正式员工')).show(3)
df.withColumn("补贴金额",lit('300')).show(3)

df.printSchema()
 df.withColumn("身份",lit('正式员工')).withColumn("补贴金额",lit('300')).printSchema()

4、去除空格

复制代码

from pyspark.sql.functions import trim,ltrim,rtrim,col

df_new = df.withColumn("new_name",concat(lit(" "),df['first_name'],lit(" ")))
 df_new.show()
df_new.select(
    "new_name",
     format_string("#%s#",col("new_name")).alias("source"),
     format_string("#%s#", ltrim(col("new_name"))).alias("ltrim_result"),
     format_string("#%s#", rtrim(col("new_name"))).alias("rtrim_result"),
     format_string("#%s#", trim(col("new_name"))).alias("trim_result")
 ).show(5)

5、正则提取

复制代码

from pyspark.sql.functions import regexp_extract

 df.select(
    'birthday',
    regexp_extract(df['birthday'],r'(\d+)-(\d+)-(\d+)',1).alias('year'),
     regexp_extract(df['birthday'],r'(\d+)-(\d+)-(\d+)',2).alias('month'),
    regexp_extract(df['birthday'],r'(\d+)-(\d+)-(\d+)',3).alias('day')
 ).show(5)

6、正则替换

复制代码

from pyspark.sql.functions import regexp_replace

df_new = df.withColumn("姓名",format_string("姓%s 名%s",df['first_name'],df['last_name']))
df_new.show(5)

df_new.withColumn("清理姓名",regexp_replace(df_new['姓名'],r'姓|名| ','')).show(5)

7、提取字符串的子串

复制代码

from pyspark.sql.functions import substring

df.select(
     'first_name',
    'birthday',
    substring(df['birthday'],0,4).alias('year'),
     substring(df['birthday'],6,2).alias('month'),
    substring(df['birthday'],-2,2).alias('day'),
).show(5)

8、字符串拆分

复制代码

from pyspark.sql.functions import split,size

df_new= df.select(
    'birthday',
    split(df['birthday'],'-').alias('splits'),
)
df_new.show(5)
df_new.printSchema()

df_new.select(
    'birthday',
    'splits',
    df_new['splits'].getItem(0).alias("year"),
    df_new['splits'].getItem(1).alias("month"),
    df_new['splits'].getItem(2).alias("day"),
    size(df_new['splits']).alias('size')
).show(5)