第5章 Kafka，构建TB级异步消息系统【仿牛客网社区论坛项目】
- 前言
- 推荐
- 项目总结
-
- [第5章 Kafka，构建TB级异步消息系统](#第5章 Kafka，构建TB级异步消息系统)
-
- 1.阻塞队列
- [2. Kafka入门](#2. Kafka入门)
- 3.Spring整合Kafka
- 4.发送系统通知
- 5.显示系统通知
- 最后
前言
2023-4-30 20:42:51
以下内容源自【Java面试项目】
仅供学习交流使用
推荐
仿牛客网项目【面试】
项目总结
第5章 Kafka，构建TB级异步消息系统
1.阻塞队列
2. Kafka入门
3.Spring整合Kafka
4.发送系统通知
java
package com.jsss.community.event;
import com.alibaba.fastjson.JSONObject;
import com.jsss.community.entity.Event;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;
@Component
public class EventProducer {
@Autowired
private KafkaTemplate kafkaTemplate;
// 处理事件
public void fireEvent(Event event) {
// 将事件发布到指定的主题
kafkaTemplate.send(event.getTopic(), JSONObject.toJSONString(event));
}
}
java
package com.jsss.community.event;
import com.alibaba.fastjson.JSONObject;
import com.jsss.community.entity.DiscussPost;
import com.jsss.community.entity.Event;
import com.jsss.community.entity.Message;
import com.jsss.community.service.DiscussPostService;
import com.jsss.community.service.ElasticsearchService;
import com.jsss.community.service.MessageService;
import com.jsss.community.util.CommunityConstant;
import com.jsss.community.util.CommunityUtil;
import com.qiniu.common.QiniuException;
import com.qiniu.common.Zone;
import com.qiniu.http.Response;
import com.qiniu.storage.Configuration;
import com.qiniu.storage.UploadManager;
import com.qiniu.util.Auth;
import com.qiniu.util.StringMap;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.annotation.KafkaListeners;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Future;
@Component
public class EventConsumer implements CommunityConstant {
private static final Logger logger = LoggerFactory.getLogger(EventConsumer.class);
@Autowired
private MessageService messageService;
@Autowired
private DiscussPostService discussPostService;
@Autowired
private ElasticsearchService elasticsearchService;
@Value("${wk.image.command}")
private String wkImageCommand;
@Value("${wk.image.storage}")
private String wkImageStorage;
//七牛云
@Value("${qiniu.key.access}")
private String accessKey;
@Value("${qiniu.key.secret}")
private String secretKey;
@Value("${qiniu.bucket.share.name}")
private String shareBucketName;
//定时任务
@Autowired
private ThreadPoolTaskScheduler taskScheduler;
//消费通知事件
@KafkaListener(topics = {TOPIC_COMMENT, TOPIC_LIKE, TOPIC_FOLLOW})
public void handleCommentMessage(ConsumerRecord record) {
if (record == null || record.value() == null) {
logger.error("消息的内容为空!");
return;
}
Event event = JSONObject.parseObject(record.value().toString(), Event.class);
if (event == null) {
logger.error("消息格式错误!");
return;
}
// 发送站内通知
Message message = new Message();
message.setFromId(SYSTEM_USER_ID);
message.setToId(event.getEntityUserId());
//复用message conversation_id 做 topic
message.setConversationId(event.getTopic());
message.setCreateTime(new Date());
//拼接语句 用户nowcoder 评论了你的 帖子 , 点击查看 !
Map<String, Object> content = new HashMap<>();
content.put("userId", event.getUserId());
content.put("entityType", event.getEntityType());
content.put("entityId", event.getEntityId());
//附加信息 一律存入content
if (!event.getData().isEmpty()) {
for (Map.Entry<String, Object> entry : event.getData().entrySet()) {
content.put(entry.getKey(), entry.getValue());
}
}
message.setContent(JSONObject.toJSONString(content));
//复用message conversation_id 做 topic
//92,1,111,like,"{""entityType"":1,""entityId"":237,""postId"":237,""userId"":112}",1,2019-04-13 22:20:58
messageService.addMessage(message);
}
// 消费发帖事件
@KafkaListener(topics = {TOPIC_PUBLISH})
public void handlePublishMessage(ConsumerRecord record) {
if (record == null || record.value() == null) {
logger.error("消息的内容为空!");
return;
}
Event event = JSONObject.parseObject(record.value().toString(), Event.class);
if (event == null) {
logger.error("消息格式错误!");
return;
}
DiscussPost post = discussPostService.findDiscussPostById(event.getEntityId());
elasticsearchService.saveDiscussPost(post);
}
// 消费删帖事件
@KafkaListener(topics = {TOPIC_DELETE})
public void handleDeleteMessage(ConsumerRecord record) {
if (record == null || record.value() == null) {
logger.error("消息的内容为空!");
return;
}
Event event = JSONObject.parseObject(record.value().toString(), Event.class);
if (event == null) {
logger.error("消息格式错误!");
return;
}
elasticsearchService.deleteDiscussPost(event.getEntityId());
}
// 消费分享事件
@KafkaListener(topics = TOPIC_SHARE)
public void handleShareMessage(ConsumerRecord record) {
if (record == null || record.value() == null) {
logger.error("消息的内容为空!");
return;
}
Event event = JSONObject.parseObject(record.value().toString(), Event.class);
if (event == null) {
logger.error("消息格式错误!");
return;
}
String htmlUrl = (String) event.getData().get("htmlUrl");
String fileName = (String) event.getData().get("fileName");
String suffix = (String) event.getData().get("suffix");
String cmd = wkImageCommand + " --quality 75 "
+ htmlUrl + " " + wkImageStorage + "/" + fileName + suffix;
try {
Runtime.getRuntime().exec(cmd);
logger.info("生成长图成功: " + cmd);
} catch (IOException e) {
logger.error("生成长图失败: " + e.getMessage());
}
// 启用定时器,监视该图片,一旦生成了,则上传至七牛云.
UploadTask task = new UploadTask(fileName, suffix);
Future future = taskScheduler.scheduleAtFixedRate(task, 500);
task.setFuture(future);
}
class UploadTask implements Runnable {
// 文件名称
private String fileName;
// 文件后缀
private String suffix;
// 启动任务的返回值
private Future future;
// 开始时间
private long startTime;
// 上传次数
private int uploadTimes;
public UploadTask(String fileName, String suffix) {
this.fileName = fileName;
this.suffix = suffix;
this.startTime = System.currentTimeMillis();
}
public void setFuture(Future future) {
this.future = future;
}
@Override
public void run() {
// 生成失败
if (System.currentTimeMillis() - startTime > 30000) {
logger.error("执行时间过长,终止任务:" + fileName);
future.cancel(true);
return;
}
// 上传失败
if (uploadTimes >= 3) {
logger.error("上传次数过多,终止任务:" + fileName);
future.cancel(true);
return;
}
String path = wkImageStorage + "/" + fileName + suffix;
File file = new File(path);
if (file.exists()) {
logger.info(String.format("开始第%d次上传[%s].", ++uploadTimes, fileName));
// 设置响应信息
StringMap policy = new StringMap();
policy.put("returnBody", CommunityUtil.getJSONString(0));
// 生成上传凭证
Auth auth = Auth.create(accessKey, secretKey);
String uploadToken = auth.uploadToken(shareBucketName, fileName, 3600, policy);
// 指定上传机房
UploadManager manager = new UploadManager(new Configuration(Zone.zone1()));
try {
// 开始上传图片
Response response = manager.put(
path, fileName, uploadToken, null, "image/" + suffix, false);
// 处理响应结果
JSONObject json = JSONObject.parseObject(response.bodyString());
if (json == null || json.get("code") == null || !json.get("code").toString().equals("0")) {
logger.info(String.format("第%d次上传失败[%s].", uploadTimes, fileName));
} else {
logger.info(String.format("第%d次上传成功[%s].", uploadTimes, fileName));
future.cancel(true);
}
} catch (QiniuException e) {
logger.info(String.format("第%d次上传失败[%s].", uploadTimes, fileName));
}
} else {
logger.info("等待图片生成[" + fileName + "].");
}
}
}
}5.显示系统通知
java
// 得到通知列表
@RequestMapping(path = "/notice/list", method = RequestMethod.GET)
public String getNoticeList(Model model) {
User user = hostHolder.getUser();
// 查询评论类通知
Message message = messageService.findLatestNotice(user.getId(), TOPIC_COMMENT);
if (message != null) {
Map<String, Object> messageVO = new HashMap<>();
messageVO.put("message", message);
//还原comment
String content = HtmlUtils.htmlUnescape(message.getContent());
Map<String, Object> data = JSONObject.parseObject(content, HashMap.class);
messageVO.put("user", userService.findUserById((Integer) data.get("userId")));
messageVO.put("entityType", data.get("entityType"));
messageVO.put("entityId", data.get("entityId"));
messageVO.put("postId", data.get("postId"));
int count = messageService.findNoticeCount(user.getId(), TOPIC_COMMENT);
messageVO.put("count", count);
int unread = messageService.findNoticeUnreadCount(user.getId(), TOPIC_COMMENT);
messageVO.put("unread", unread);
model.addAttribute("commentNotice", messageVO);
}
// 查询点赞类通知
message = messageService.findLatestNotice(user.getId(), TOPIC_LIKE);
if (message != null) {
Map<String, Object> messageVO = new HashMap<>();
messageVO.put("message", message);
String content = HtmlUtils.htmlUnescape(message.getContent());
Map<String, Object> data = JSONObject.parseObject(content, HashMap.class);
messageVO.put("user", userService.findUserById((Integer) data.get("userId")));
messageVO.put("entityType", data.get("entityType"));
messageVO.put("entityId", data.get("entityId"));
messageVO.put("postId", data.get("postId"));
int count = messageService.findNoticeCount(user.getId(), TOPIC_LIKE);
messageVO.put("count", count);
int unread = messageService.findNoticeUnreadCount(user.getId(), TOPIC_LIKE);
messageVO.put("unread", unread);
model.addAttribute("likeNotice", messageVO);
}
// 查询关注类通知
message = messageService.findLatestNotice(user.getId(), TOPIC_FOLLOW);
if (message != null) {
Map<String, Object> messageVO = new HashMap<>();
messageVO.put("message", message);
String content = HtmlUtils.htmlUnescape(message.getContent());
Map<String, Object> data = JSONObject.parseObject(content, HashMap.class);
messageVO.put("user", userService.findUserById((Integer) data.get("userId")));
messageVO.put("entityType", data.get("entityType"));
messageVO.put("entityId", data.get("entityId"));
int count = messageService.findNoticeCount(user.getId(), TOPIC_FOLLOW);
messageVO.put("count", count);
int unread = messageService.findNoticeUnreadCount(user.getId(), TOPIC_FOLLOW);
messageVO.put("unread", unread);
model.addAttribute("followNotice", messageVO);
}
// 查询未读消息数量
int letterUnreadCount = messageService.findLetterUnreadCount(user.getId(), null);
model.addAttribute("letterUnreadCount", letterUnreadCount);
int noticeUnreadCount = messageService.findNoticeUnreadCount(user.getId(), null);
model.addAttribute("noticeUnreadCount", noticeUnreadCount);
return "site/notice";
}
// 得到通知详情
@RequestMapping(path = "/notice/detail/{topic}", method = RequestMethod.GET)
public String getNoticeDetail(@PathVariable("topic") String topic, Page page, Model model) {
User user = hostHolder.getUser();
//分页信息
page.setLimit(5);
page.setPath("/notice/detail/" + topic);
page.setRows(messageService.findNoticeCount(user.getId(), topic));
//通知信息
List<Message> noticeList = messageService.findNotices(user.getId(), topic, page.getOffset(), page.getLimit());
List<Map<String, Object>> noticeVoList = new ArrayList<>();
if (noticeList != null) {
for (Message notice : noticeList) {
Map<String, Object> map = new HashMap<>();
// 通知
map.put("notice", notice);
// 内容
String content = HtmlUtils.htmlUnescape(notice.getContent());
Map<String, Object> data = JSONObject.parseObject(content, HashMap.class);
map.put("user", userService.findUserById((Integer) data.get("userId")));
map.put("entityType", data.get("entityType"));
map.put("entityId", data.get("entityId"));
map.put("postId", data.get("postId"));
// 通知作者
map.put("fromUser", userService.findUserById(notice.getFromId()));
noticeVoList.add(map);
}
}
model.addAttribute("notices", noticeVoList);
// 设置已读
List<Integer> ids = getLetterIds(noticeList);
if (!ids.isEmpty()) {
messageService.readMessage(ids);
}
return "site/notice-detail";
}最后
这篇博客能写好的原因是：站在巨人的肩膀上
这篇博客要写好的目的是：做别人的肩膀
开源：为爱发电
学习：为我而行