搜索引擎日志分析
需求1 用户搜索关键词分析
需求2 用户和关键词组合分析
需求3 热门搜索时间段分析
首先对于数据可以用jieba库进行分词处理
python
from pyspark import SparkConf,SparkContext
from defs import context_jieba,filter_words,append_words
from oprator import add
from pyspark.storagelevel import storagelevel
if __name__ == '__main__':
conf = SparkConf().setAppName('sougou').setMaster("local[*]")
#读取文件
sc = SparkContext('../sougou.txt')
split_rdd = file_rdd.map(lambda line:line.split('\t'))
#送入缓存
split_rdd.persist(StorageLevel.DISK_ONLY)
#需求1
context_rdd = split_rdd.map(lambda x : x[2])
# 对取出来的第二列搜索内容做分词操作
word_rdd = context_rdd.flatMap(context_jieba)
filter_rdd = word_rdd.filter(filter_words)
final_words_rdd = filter_rdd.amp(append_words)
#统计关键词的个数
result1 = final_words_rdd.reduceByKey(lambda a,b: a + b).sortBy(lambda x : x[1],ascending=False,numPartitions=1).take(5)
print('需求一结果',result1)
#需求2
user_content_rdd = split_rdd.map(lambda x:(x[1],x[2]))
user_word_rdd = uder_content_tdd.flatMap(extract_user_and_word)
user_word_with_one_rddd = user_word_rdd.map(lambda x:(x:1))
result2 = user_word_with_one_rddd.reduceByKey(lambda a,b:a+b).sortBy(lambda x:x[1],ascending=False,numpPartitions=1).take(5)
print(result2)
#需求3
time_rdd = split_rdd.map(lambda x:x[0])
hour_with_one_rdd = time_rdd.map(lambda x:(x.split(":")[0],1))
result3 = hpour_with_one_rdd.reduceByKey(add).sortBy(lambda x:x[1],ascending=False,numPartitions=1).collect()
print(result3)将代码提交到集群中运行
powershell
/spark/bin/spark-submit --master yarn --py-files def.py /root/main.py