generate_file.py
生成数据 生成500个文件,每个文件1000条数据
生成数据格式：eventtime name province action ()时间 用户名 省份 动作)
import os
import time
import shutil
import time
FIRST_NAME = 'Zhao', 'Qian', 'Sun', 'Li', 'Zhou', 'Wu', 'Zheng', 'Wang'
SECOND_NAME = 'San', 'Si', 'Wu', 'Chen', 'Yang', 'Min', 'Jie', 'Qi'
PROVINCE = 'BeiJing', 'ShanDong', 'ShangHai', 'HeNan', 'HaErBin'
ACTION = 'login', 'logout', 'purchase'
PATH = "/opt/software/tmp/"
DATA_PATH = "/opt/software/tmp/data/"
初始化环境
def test_Setup():
if os.path.exists(DATA_PATH):
shutil.rmtree(DATA_PATH)
os.mkdir(DATA_PATH)
清理数据，恢复测试环境
def test_TearDown():
shutile.rmtree(DATA_PATH)
数据保存文件
def writeAndMove(filename,content):
with open(PATH+filename,'wt',encoding='utf-8') as f:
f.write(content)
shutil.move(PATH+filename,DATA_PATH+filename)
if name == 'main':
test_Setup()
for i in range(500):
filename = "user_action_{}.log".format(i)
"""
验证spark输出模式，complete和update,增加代码，第一个文件i=0时，设置PROVINCE = "TAIWAN"
"""
if i == 0:
province= 'TaiWan'
else:
province = PROVINCE
content = ""
for _ in range(1000):
content += "{} {} {} {}\n".format(str(int(time.time())),random.choice(FIRST_NAME)+random.choice(SECOND_NAME),random.choice(province),random.choice(ACTION))
writeAndMove(filename,content)
time.sleep(10)
spark_file_test.py
读取DATA文件夹下面文件，按照省份统计数据，主要考虑window情况，按照window情况测试，同时针对 outputMode和输出console和mysql进行考虑，其中保存到mysql时添加batch字段
from pyspark.sql import SparkSession,DataFrame
from pyspark.sql.functions import split,lit,from_unixtime
DATA_PATH = "/opt/software/tmp/data/"
if name == 'main':
spark = SparkSession.builder.getOrCreate()
lines = spark.readStream.format("text").option("seq","\n").load(DATA_PATH)
分隔符为空格
userinfo = lines.select(split(lines.value," ").alias("info"))
第一个为eventtime 第二个为name 第三个为province 第四个为action
userinfo'info'0等同于userinfo'info'.getIterm(0)
user = userinfo.select(from_unixtime(userinfo'info'0).alias('eventtime'),
userinfo'info'1.alias('name'),userinfo'info'2.alias('province'),
userinfo'info'3.alias('action'))
"""
测试1：数据直接输出到控制台，由于没有采用聚合，输出模式选择update
user.writeStream.outputMode("update").format("console").trigger(processingTime="8 seconds").start().awaitTermination()
"""
"""
测试2：数据存储到数据库，新建数据库表，可以通过printSchema()查看数据类型情况
def insert_into_mysql_batch(df:DataFrame,batch):
if df.count()>0:
此处将batch添加到df中，采用lit函数
data = df.withColumn("batch",lit(batch))
data.write.format("jdbc"). \
option("driver","com.mysql.jdbc.Driver"). \
option("url","jdbc:mysql://localhost:3306/spark").option("user","root").\
option("password","root").option("dbtable","user_log").\
option("batchsize",1000).mode("append").save()
else:
pass
user.writeStream.outputMode("update").foreachBatch((insert_into_mysql_batch)).trigger(processingTime="20 seconds").start().awaitTermination()
"""
"""
测试3：数据按照省份统计后，输出到控制台，分析complete和update输出模式区别,针对该问题，调整输入，province="TaiWan"只会输入1次，即如果输出方式complete,则每batch都会输出，update的话，只会出现在一个batch
userProvinceCounts = user.groupBy("province").count()
userProvinceCounts = userProvinceCounts.select(userProvinceCounts'province',userProvinceCounts"count".alias('sl'))
测试输出模式complete:complete将总计算结果都进行输出
"""
batch 0
TaiWan 1000
batch 1
TaiWan 1000
其他省份 sl
batch 2
TaiWan 1000
其他省份 sl
""" userProvinceCounts.writeStream.outputMode("complete").format("console").trigger(processingTime="20 seconds").start().awaitTermination()
测试输出模式update:update只输出相比上个批次变动的内容(新增或修改)
batch 0
TaiWan 1000
batch 1 中没有TaiWan输出
userProvinceCounts.writeStream.outputMode("complete").format("console").trigger(processingTime="20 seconds").start().awaitTermination()
"""