省流
该文从官网消费者示例开始，分析Pull模式和Push模式的消费过程，broker如何处理mq消费请求下一篇见，具体的逻辑代码可通过侧边导航快速定位
客户端获取消息
先来看看官网的快速开始示例PushConsumer，消费者一共分类有以下三种
| 对比项 | PushConsumer | SimpleConsumer | PullConsumer |
|---|---|---|---|
| 接口方式 | 使用监听器回调接口返回消费结果，消费者仅允许在监听器范围内处理消费逻辑。 | 业务方自行实现消息处理，并主动调用接口返回消费结果。 | 业务方自行按队列拉取消息，并可选择性地提交消费结果 |
| 消费并发度管理 | 由SDK管理消费并发度。 | 由业务方消费逻辑自行管理消费线程。 | 由业务方消费逻辑自行管理消费线程。 |
| 负载均衡粒度 | 5.0 SDK是消息粒度，更均衡，早期版本是队列维度 | 消息粒度，更均衡 | 队列粒度，吞吐攒批性能更好，但容易不均衡 |
| 接口灵活度 | 高度封装，不够灵活。 | 原子接口，可灵活自定义。 | 原子接口，可灵活自定义。 |
| 适用场景 | 适用于无自定义流程的业务消息开发场景。 | 适用于需要高度自定义业务流程的业务开发场景。 | 仅推荐在流处理框架场景下集成使用 |
但是这里需要注意的是
生产环境中相同的 ConsumerGroup 下严禁混用 PullConsumer 和其他两种消费者，否则会导致消息消费异常。
PushConsumer
官网PushConsumer示例
java
import java.io.IOException;
import java.util.Collections;
import org.apache.rocketmq.client.apis.ClientConfiguration;
import org.apache.rocketmq.client.apis.ClientException;
import org.apache.rocketmq.client.apis.ClientServiceProvider;
import org.apache.rocketmq.client.apis.consumer.ConsumeResult;
import org.apache.rocketmq.client.apis.consumer.FilterExpression;
import org.apache.rocketmq.client.apis.consumer.FilterExpressionType;
import org.apache.rocketmq.client.apis.consumer.PushConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PushConsumerExample {
private static final Logger logger = LoggerFactory.getLogger(PushConsumerExample.class);
private PushConsumerExample() {
}
public static void main(String[] args) throws ClientException, IOException, InterruptedException {
final ClientServiceProvider provider = ClientServiceProvider.loadService();
// 接入点地址，需要设置成Proxy的地址和端口列表，一般是xxx:8081;xxx:8081。
String endpoints = "localhost:8081";
ClientConfiguration clientConfiguration = ClientConfiguration.newBuilder()
.setEndpoints(endpoints)
.build();
// 订阅消息的过滤规则，表示订阅所有Tag的消息。
String tag = "*";
FilterExpression filterExpression = new FilterExpression(tag, FilterExpressionType.TAG);
// 为消费者指定所属的消费者分组，Group需要提前创建。
String consumerGroup = "YourConsumerGroup";
// 指定需要订阅哪个目标Topic，Topic需要提前创建。
String topic = "TestTopic";
// 初始化PushConsumer，需要绑定消费者分组ConsumerGroup、通信参数以及订阅关系。
PushConsumer pushConsumer = provider.newPushConsumerBuilder()
.setClientConfiguration(clientConfiguration)
// 设置消费者分组。
.setConsumerGroup(consumerGroup)
// 设置预绑定的订阅关系。
.setSubscriptionExpressions(Collections.singletonMap(topic, filterExpression))
// 设置消费监听器。
.setMessageListener(messageView -> {
// 处理消息并返回消费结果。
logger.info("Consume message successfully, messageId={}", messageView.getMessageId());
return ConsumeResult.SUCCESS;
})
.build();
Thread.sleep(Long.MAX_VALUE);
// 如果不需要再使用 PushConsumer，可关闭该实例。
// pushConsumer.close();
}
}如何构建PushConsumer
这里对PushConsumer进行配置后，通过构造器build出消费者对象，来看看build做了什么
java
/**
* @see PushConsumerBuilder#build()
*/
@Override
public PushConsumer build() throws ClientException {
//校验配置信息
checkNotNull(clientConfiguration, "clientConfiguration has not been set yet");
checkNotNull(consumerGroup, "consumerGroup has not been set yet");
checkNotNull(messageListener, "messageListener has not been set yet");
checkArgument(!subscriptionExpressions.isEmpty(), "subscriptionExpressions have not been set yet");
//构建消费者 ，push
final PushConsumerImpl pushConsumer = new PushConsumerImpl(clientConfiguration, consumerGroup,
subscriptionExpressions, messageListener, maxCacheMessageCount, maxCacheMessageSizeInBytes,
consumptionThreadCount);
pushConsumer.startAsync().awaitRunning();
return pushConsumer;
}这里对配置进行了检验后创建他的实现类并启动他，来看看PushConsumerImpl对象的相关方法
java
/**
* The caller is supposed to have validated the arguments and handled throwing exception or
* logging warnings already, so we avoid repeating args check here.
* 调用者应该已经验证了参数并处理了抛出的异常或日志警告，因此我们避免在这里重复args检查。
*/
public PushConsumerImpl(ClientConfiguration clientConfiguration, String consumerGroup,
Map<String, FilterExpression> subscriptionExpressions, MessageListener messageListener,
int maxCacheMessageCount, int maxCacheMessageSizeInBytes, int consumptionThreadCount) {
//构建ConsumerImpl
super(clientConfiguration, consumerGroup, subscriptionExpressions.keySet());
this.clientConfiguration = clientConfiguration;
Resource groupResource = new Resource(clientConfiguration.getNamespace(), consumerGroup);
//发布配置
this.pushSubscriptionSettings = new PushSubscriptionSettings(clientConfiguration.getNamespace(), clientId,
endpoints, groupResource, clientConfiguration.getRequestTimeout(), subscriptionExpressions);
//所属消费者组
this.consumerGroup = consumerGroup;
//订阅配置
this.subscriptionExpressions = subscriptionExpressions;
this.cacheAssignments = new ConcurrentHashMap<>();
//消息监听器
this.messageListener = messageListener;
//最大缓存的消息数和消息大小
this.maxCacheMessageCount = maxCacheMessageCount;
this.maxCacheMessageSizeInBytes = maxCacheMessageSizeInBytes;
//消息接收次数
this.receptionTimes = new AtomicLong(0);
//消息接收数量
this.receivedMessagesQuantity = new AtomicLong(0);
// 消费者的 ok error 计数器
this.consumptionOkQuantity = new AtomicLong(0);
this.consumptionErrorQuantity = new AtomicLong(0);
//处理的队列表
this.processQueueTable = new ConcurrentHashMap<>();
// 消费者线程池
this.consumptionExecutor = new ThreadPoolExecutor(
consumptionThreadCount,
consumptionThreadCount,
60,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
new ThreadFactoryImpl("MessageConsumption", this.getClientId().getIndex()));
}
@Override
protected void startUp() throws Exception {
try {
log.info("Begin to start the rocketmq push consumer, clientId={}", clientId);
GaugeObserver gaugeObserver = new ProcessQueueGaugeObserver(processQueueTable, clientId, consumerGroup);
this.clientMeterManager.setGaugeObserver(gaugeObserver);
//调用ClientImpl的startUp方法，该方法会去刷新topic路由
super.startUp();
final ScheduledExecutorService scheduler = this.getClientManager().getScheduler();
//创建消费服务
this.consumeService = createConsumeService();
// 延迟1秒，每隔5秒去scanAssignments一次（队列信息，在4.X版本是在rebalance时调用，具体的是通过rpc调用服务）
//具体的就是去拉 接入点的队列，并进行处理 目的是同步队列信息
scanAssignmentsFuture = scheduler.scheduleWithFixedDelay(() -> {
try {
scanAssignments();
} catch (Throwable t) {
log.error("Exception raised while scanning the load assignments, clientId={}", clientId, t);
}
}, 1, 5, TimeUnit.SECONDS);
log.info("The rocketmq push consumer starts successfully, clientId={}", clientId);
} catch (Throwable t) {
log.error("Exception raised while starting the rocketmq push consumer, clientId={}", clientId, t);
shutDown();
throw t;
}
}来看看createConsumeService()做了什么
java
private ConsumeService createConsumeService() {
final ScheduledExecutorService scheduler = this.getClientManager().getScheduler();
//是否fifo消费者
if (pushSubscriptionSettings.isFifo()) {
log.info("Create FIFO consume service, consumerGroup={}, clientId={}", consumerGroup, clientId);
return new FifoConsumeService(clientId, messageListener, consumptionExecutor, this, scheduler);
}
//标准的消费者
log.info("Create standard consume service, consumerGroup={}, clientId={}", consumerGroup, clientId);
return new StandardConsumeService(clientId, messageListener, consumptionExecutor, this, scheduler);
}这里我们按StandardConsumeService进行分析
java
public class StandardConsumeService extends ConsumeService {
private static final Logger log = LoggerFactory.getLogger(StandardConsumeService.class);
public StandardConsumeService(ClientId clientId, MessageListener messageListener,
ThreadPoolExecutor consumptionExecutor, MessageInterceptor messageInterceptor,
ScheduledExecutorService scheduler) {
super(clientId, messageListener, consumptionExecutor, messageInterceptor, scheduler);
}
@Override
public void consume(ProcessQueue pq, List<MessageViewImpl> messageViews) {
for (MessageViewImpl messageView : messageViews) {
// Discard corrupted message.
if (messageView.isCorrupted()) {
log.error("Message is corrupted for standard consumption, prepare to discard it, mq={}, "
+ "messageId={}, clientId={}", pq.getMessageQueue(), messageView.getMessageId(), clientId);
pq.discardMessage(messageView);
continue;
}
//调用父类ConsumeService的consume(MessageViewImpl)方法
final ListenableFuture<ConsumeResult> future = consume(messageView);
Futures.addCallback(future, new FutureCallback<ConsumeResult>() {
@Override
public void onSuccess(ConsumeResult consumeResult) {
//处理消息ack和本地消息缓存
pq.eraseMessage(messageView, consumeResult);
}
@Override
public void onFailure(Throwable t) {
// Should never reach here.
log.error("[Bug] Exception raised in consumption callback, clientId={}", clientId, t);
}
}, MoreExecutors.directExecutor());
}
}
}PushConsumer怎么去消费消息
StandardConsumeService继承了ConsumeService并重写了consume方法，这里主要的消费逻辑是父类ConsumeService的consume(MessageViewImpl)
java
public ListenableFuture<ConsumeResult> consume(MessageViewImpl messageView, Duration delay) {
final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(consumptionExecutor);
final ConsumeTask task = new ConsumeTask(clientId, messageListener, messageView, messageInterceptor);
// 立刻消费
if (Duration.ZERO.compareTo(delay) >= 0) {
//立刻提交给消费者线程池处理消息
return executorService.submit(task);
}
//延迟消费
final SettableFuture<ConsumeResult> future0 = SettableFuture.create();
scheduler.schedule(() -> {
final ListenableFuture<ConsumeResult> future = executorService.submit(task);
Futures.addCallback(future, new FutureCallback<ConsumeResult>() {
@Override
public void onSuccess(ConsumeResult consumeResult) {
future0.set(consumeResult);
}
@Override
public void onFailure(Throwable t) {
// Should never reach here.
log.error("[Bug] Exception raised while submitting scheduled consumption task, clientId={}",
clientId, t);
}
}, MoreExecutors.directExecutor());
}, delay.toNanos(), TimeUnit.NANOSECONDS);
return future0;
}这里根据消息类型，创建消费任务提交到执行线程池中处理，来看看ConsumeTask的call方法，消费消息做了什么
java
public ListenableFuture<ConsumeResult> consume(MessageViewImpl messageView, Duration delay) {
final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(consumptionExecutor);
final ConsumeTask task = new ConsumeTask(clientId, messageListener, messageView, messageInterceptor);
// 立刻消费
if (Duration.ZERO.compareTo(delay) >= 0) {
//立刻提交给消费者线程池处理消息
return executorService.submit(task);
}
//延迟消费
final SettableFuture<ConsumeResult> future0 = SettableFuture.create();
scheduler.schedule(() -> {
final ListenableFuture<ConsumeResult> future = executorService.submit(task);
Futures.addCallback(future, new FutureCallback<ConsumeResult>() {
@Override
public void onSuccess(ConsumeResult consumeResult) {
future0.set(consumeResult);
}
@Override
public void onFailure(Throwable t) {
// Should never reach here.
log.error("[Bug] Exception raised while submitting scheduled consumption task, clientId={}",
clientId, t);
}
}, MoreExecutors.directExecutor());
}, delay.toNanos(), TimeUnit.NANOSECONDS);
return future0;
}ConsumeTask是个线程类，来看看他的call方法
java
/**
* Invoke {@link MessageListener} to consumer message.
*
* @return message(s) which is consumed successfully.
*/
@Override
public ConsumeResult call() {
ConsumeResult consumeResult;
//处理消息处理前置拦截器
final List<GeneralMessage> generalMessages = Collections.singletonList(new GeneralMessageImpl(messageView));
MessageInterceptorContextImpl context = new MessageInterceptorContextImpl(MessageHookPoints.CONSUME);
messageInterceptor.doBefore(context, generalMessages);
//调用创建的Listen进行消息消费
try {
consumeResult = messageListener.consume(messageView);
} catch (Throwable t) {
log.error("Message listener raised an exception while consuming messages, clientId={}, mq={}, " +
"messageId={}", clientId, messageView.getMessageQueue(), messageView.getMessageId(), t);
// If exception was thrown during the period of message consumption, mark it as failure.
consumeResult = ConsumeResult.FAILURE;
}
//计算消费结果
MessageHookPointsStatus status = ConsumeResult.SUCCESS.equals(consumeResult) ? MessageHookPointsStatus.OK :
MessageHookPointsStatus.ERROR;
context = new MessageInterceptorContextImpl(context, status);
//后置拦截器
messageInterceptor.doAfter(context, generalMessages);
// Make sure that the return value is the subset of messageViews.
return consumeResult;
}PushConsumer获取消息
从上面我们知道了怎么消费消息，我们来看看怎么获取消息，在PushConsumerImpl#startUp中,创建完消费者服务后，有个定时任务会去每隔5秒获取调用一次scanAssignments();,来看看他做了什么
java
void scanAssignments() {
try {
log.debug("Start to scan assignments periodically, clientId={}", clientId);
for (Map.Entry<String, FilterExpression> entry : subscriptionExpressions.entrySet()) {
final String topic = entry.getKey();
final FilterExpression filterExpression = entry.getValue();
final Assignments existed = cacheAssignments.get(topic);
//构建根据topic获取assignments的请求
final ListenableFuture<Assignments> future = queryAssignment(topic);
Futures.addCallback(future, new FutureCallback<Assignments>() {
@Override
public void onSuccess(Assignments latest) {
if (latest.getAssignmentList().isEmpty()) {
if (null == existed || existed.getAssignmentList().isEmpty()) {
log.info("Acquired empty assignments from remote, would scan later, topic={}, "
+ "clientId={}", topic, clientId);
return;
}
log.info("Attention!!! acquired empty assignments from remote, but existed assignments"
+ " is not empty, topic={}, clientId={}", topic, clientId);
}
if (!latest.equals(existed)) {
log.info("Assignments of topic={} has changed, {} => {}, clientId={}", topic, existed,
latest, clientId);
//在这里 如果队列数据不对等，会去处理队列
syncProcessQueue(topic, latest, filterExpression);
cacheAssignments.put(topic, latest);
return;
}
log.debug("Assignments of topic={} remains the same, assignments={}, clientId={}", topic,
existed, clientId);
// 进程队列可能被丢弃，无论如何都需要同步
syncProcessQueue(topic, latest, filterExpression);
}
@Override
public void onFailure(Throwable t) {
log.error("Exception raised while scanning the assignments, topic={}, clientId={}", topic,
clientId, t);
}
}, MoreExecutors.directExecutor());
}
} catch (Throwable t) {
log.error("Exception raised while scanning the assignments for all topics, clientId={}", clientId, t);
}
}这里获取到queue信息后，会执行syncProcessQueue方法，syncProcessQueue方法做了什么？我们来看看
java
void syncProcessQueue(String topic, Assignments assignments, FilterExpression filterExpression) {
Set<MessageQueueImpl> latest = new HashSet<>();
final List<Assignment> assignmentList = assignments.getAssignmentList();
for (Assignment assignment : assignmentList) {
latest.add(assignment.getMessageQueue());
}
Set<MessageQueueImpl> activeMqs = new HashSet<>();
//判断队列表中的队列是否健康
for (Map.Entry<MessageQueueImpl, ProcessQueue> entry : processQueueTable.entrySet()) {
final MessageQueueImpl mq = entry.getKey();
final ProcessQueue pq = entry.getValue();
if (!topic.equals(mq.getTopic())) {
continue;
}
if (!latest.contains(mq)) {
log.info("Drop message queue according to the latest assignmentList, mq={}, clientId={}", mq,
clientId);
dropProcessQueue(mq);
continue;
}
if (pq.expired()) {
log.warn("Drop message queue because it is expired, mq={}, clientId={}", mq, clientId);
dropProcessQueue(mq);
continue;
}
activeMqs.add(mq);
}
//把不在本地队列表中的队列加入到对列表中,并调用ProcessQueue的#fetchMessageImmediately
for (MessageQueueImpl mq : latest) {
if (activeMqs.contains(mq)) {
continue;
}
final Optional<ProcessQueue> optionalProcessQueue = createProcessQueue(mq, filterExpression);
if (optionalProcessQueue.isPresent()) {
log.info("Start to fetch message from remote, mq={}, clientId={}", mq, clientId);
optionalProcessQueue.get().fetchMessageImmediately();
}
}
}在这里，我们知道了当拉到一个新的队列的时候，会调用createProcessQueue方法构建Optional<ProcessQueue>对象，然后调用他的#fetchMessageImmediately方法，在ProcessQueue接口中，这样描述到
java
/**
* Start to fetch messages from remote immediately.
* 立即开始从远程获取消息
*/
void fetchMessageImmediately();我们来看看这个方法做了什么
java
@Override
public void fetchMessageImmediately() {
receiveMessageImmediately();
}
private void receiveMessageImmediately() {
receiveMessageImmediately(this.generateAttemptId());
}
private void receiveMessageImmediately(String attemptId) {
final ClientId clientId = consumer.getClientId();
//判断消费者是否活着
if (!consumer.isRunning()) {
log.info("Stop to receive message because consumer is not running, mq={}, clientId={}", mq, clientId);
return;
}
try {
//获取接入点信息
final Endpoints endpoints = mq.getBroker().getEndpoints();
//计算一批最多多少， 最大缓存数/队列数-已缓存的消息数 最小值为1
final int batchSize = this.getReceptionBatchSize();
//长轮询的超时时间
final Duration longPollingTimeout = consumer.getPushConsumerSettings().getLongPollingTimeout();
//构建接受消息的请求
final ReceiveMessageRequest request = consumer.wrapReceiveMessageRequest(batchSize, mq, filterExpression,
longPollingTimeout, attemptId);
activityNanoTime = System.nanoTime();
// 执行消息拦截器前置doBefore
final MessageInterceptorContextImpl context = new MessageInterceptorContextImpl(MessageHookPoints.RECEIVE);
consumer.doBefore(context, Collections.emptyList());
//构建接受消息future
final ListenableFuture<ReceiveMessageResult> future = consumer.receiveMessage(request, mq,
longPollingTimeout);
//注册回调
Futures.addCallback(future, new FutureCallback<ReceiveMessageResult>() {
@Override
public void onSuccess(ReceiveMessageResult result) {
// 执行消息后置拦截器 doAfter
final List<GeneralMessage> generalMessages = result.getMessageViewImpls().stream()
.map((Function<MessageView, GeneralMessage>) GeneralMessageImpl::new)
.collect(Collectors.toList());
final MessageInterceptorContextImpl context0 =
new MessageInterceptorContextImpl(context, MessageHookPointsStatus.OK);
consumer.doAfter(context0, generalMessages);
//处理消息
try {
onReceiveMessageResult(result);
} catch (Throwable t) {
// Should never reach here.
log.error("[Bug] Exception raised while handling receive result, mq={}, endpoints={}, "
+ "clientId={}", mq, endpoints, clientId, t);
onReceiveMessageException(t, attemptId);
}
}
@Override
public void onFailure(Throwable t) {
String nextAttemptId = null;
if (t instanceof StatusRuntimeException) {
StatusRuntimeException exception = (StatusRuntimeException) t;
if (io.grpc.Status.DEADLINE_EXCEEDED.getCode() == exception.getStatus().getCode()) {
nextAttemptId = request.getAttemptId();
}
}
// 执行消息后置拦截器 doAfter
final MessageInterceptorContextImpl context0 =
new MessageInterceptorContextImpl(context, MessageHookPointsStatus.ERROR);
consumer.doAfter(context0, Collections.emptyList());
log.error("Exception raised during message reception, mq={}, endpoints={}, attemptId={}, " +
"nextAttemptId={}, clientId={}", mq, endpoints, request.getAttemptId(), nextAttemptId,
clientId, t);
//处理接受异常，具体的操作是 延迟再接收
onReceiveMessageException(t, nextAttemptId);
}
}, MoreExecutors.directExecutor());
receptionTimes.getAndIncrement();
consumer.getReceptionTimes().getAndIncrement();
} catch (Throwable t) {
log.error("Exception raised during message reception, mq={}, clientId={}", mq, clientId, t);
onReceiveMessageException(t, attemptId);
}
}
private void onReceiveMessageResult(ReceiveMessageResult result) {
final List<MessageViewImpl> messages = result.getMessageViewImpls();
if (!messages.isEmpty()) {
//缓存消息并计算已经缓存的消息大小
cacheMessages(messages);
//更新接收到的消息计数器
receivedMessagesQuantity.getAndAdd(messages.size());
//更新消费者已经收到的消息计数器
consumer.getReceivedMessagesQuantity().getAndAdd(messages.size());
//进行消费
consumer.getConsumeService().consume(this, messages);
}
//再次接受消息
receiveMessage();
}
public void receiveMessage() {
receiveMessage(this.generateAttemptId());
}从源码中我们可知消息是通过长轮询mq获取消息，收到消息后将消息交给消费者后更新相关计数器就继续重新接收消息
PullConsumer
构建一个PullConsumer
在4.x版本的官网文档上提供了这样一个示例
java
DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5");
consumer.setNamesrvAddr("127.0.0.1:9876");
Set<String> topics = new HashSet<>();
//You would be better to register topics,It will use in rebalance when starting
topics.add("TopicTest");
consumer.setRegisterTopics(topics);
consumer.start();
ExecutorService executors = Executors.newFixedThreadPool(topics.size(), new ThreadFactoryImpl("PullConsumerThread"));
for (String topic : consumer.getRegisterTopics()) {
executors.execute(new Runnable() {
@Override
public void run() {
while (true) {
try {
Set<MessageQueue> messageQueues = consumer.fetchMessageQueuesInBalance(topic);
if (messageQueues == null || messageQueues.isEmpty()) {
Thread.sleep(1000);
continue;
}
PullResult pullResult = null;
for (MessageQueue messageQueue : messageQueues) {
try {
long offset = this.consumeFromOffset(messageQueue);
pullResult = consumer.pull(messageQueue, "*", offset, 32);
switch (pullResult.getPullStatus()) {
case FOUND:
List<MessageExt> msgs = pullResult.getMsgFoundList();
if (msgs != null && !msgs.isEmpty()) {
//消费消息
//更新本地offset, eventually to broker
consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
//print pull tps
this.incPullTPS(topic, pullResult.getMsgFoundList().size());
}
break;
//错误的offset
case OFFSET_ILLEGAL:
consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
break;
//没有新消息
case NO_NEW_MSG:
Thread.sleep(1);
consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
break;
//不符合过滤的结果
case NO_MATCHED_MSG:
consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
break;
default:
}
} catch (Exception e){
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
public long consumeFromOffset(MessageQueue messageQueue) throws MQClientException {
//-1 when started
long offset = consumer.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY);
if (offset < 0) {
//query from broker
offset = consumer.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE);
}
if (offset < 0) {
//first time start from last offset
offset = consumer.maxOffset(messageQueue);
}
//make sure
if (offset < 0) {
offset = 0;
}
return offset;
}
public void incPullTPS(String topic, int pullSize) {
consumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory()
.getConsumerStatsManager().incPullTPS(consumer.getConsumerGroup(), topic, pullSize);
}
});
}
Thread.sleep(Long.MAX_VALUE);
executors.shutdown();
consumer.shutdown();构建PullConsumer实例之后，通过setRegisterTopics设置他监听的topic，然后调用start方法启动
java
public synchronized void start() throws MQClientException {
switch (this.serviceState) {//服务状态
//刚创建
case CREATE_JUST:
this.serviceState = ServiceState.START_FAILED;
this.checkConfig();//校验配置
this.copySubscription();//拷贝订阅信息
if (this.defaultMQPullConsumer.getMessageModel() == MessageModel.CLUSTERING) {//是否集群
this.defaultMQPullConsumer.changeInstanceNameToPID();
}
//这里的 defaultMQPullConsumer是DefaultMQPullConsumerImpl
this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQPullConsumer, this.rpcHook);//创建clientFactory
this.rebalanceImpl.setConsumerGroup(this.defaultMQPullConsumer.getConsumerGroup());//设置消费者组
this.rebalanceImpl.setMessageModel(this.defaultMQPullConsumer.getMessageModel());//消息类型
this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPullConsumer.getAllocateMessageQueueStrategy());//配置消息分配算法
this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);
this.pullAPIWrapper = new PullAPIWrapper(
mQClientFactory,
this.defaultMQPullConsumer.getConsumerGroup(), isUnitMode());
this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);
if (this.defaultMQPullConsumer.getOffsetStore() != null) {
this.offsetStore = this.defaultMQPullConsumer.getOffsetStore();
} else {
switch (this.defaultMQPullConsumer.getMessageModel()) {
//广播
case BROADCASTING:
this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPullConsumer.getConsumerGroup());
break;
//集群
case CLUSTERING:
this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPullConsumer.getConsumerGroup());
break;
default:
break;
}
this.defaultMQPullConsumer.setOffsetStore(this.offsetStore);
}
this.offsetStore.load();
//注册消费者
boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPullConsumer.getConsumerGroup(), this);
if (!registerOK) {//注册失败
this.serviceState = ServiceState.CREATE_JUST;
throw new MQClientException("The consumer group[" + this.defaultMQPullConsumer.getConsumerGroup()
+ "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),
null);
}
mQClientFactory.start();//启动client
log.info("the consumer [{}] start OK", this.defaultMQPullConsumer.getConsumerGroup());
this.serviceState = ServiceState.RUNNING;
break;
case RUNNING://运行中
case START_FAILED://运行失败
case SHUTDOWN_ALREADY://关闭
throw new MQClientException("The PullConsumer service state not OK, maybe started once, "
+ this.serviceState
+ FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),
null);
default:
break;
}
}PullConsumer创建client
再来看看mQClientFactory.start();做了什么
java
public void start() throws MQClientException {
synchronized (this) {
switch (this.serviceState) {
case CREATE_JUST:
this.serviceState = ServiceState.START_FAILED;
// 如果未指定，则从名称服务器查找地址
if (null == this.clientConfig.getNamesrvAddr()) {
this.mQClientAPIImpl.fetchNameServerAddr();
}
// 启动请求-响应通道
this.mQClientAPIImpl.start();
// 启动各种计划任务[同步NameServer、更新Topic、心跳、更新offset、线程池监控]
this.startScheduledTask();
// 启动PullMessage服务
this.pullMessageService.start();
// 启动重平衡服务
this.rebalanceService.start();
// 启动生产消息的服务
this.defaultMQProducer.getDefaultMQProducerImpl().start(false);
log.info("the client factory [{}] start OK", this.clientId);
this.serviceState = ServiceState.RUNNING;
break;
case START_FAILED:
throw new MQClientException("The Factory object[" + this.getClientId() + "] has been created before, and failed.", null);
default:
break;
}
}
}PullConsumer如何去获取消息
这里获取消息的服务是pullMessageService，来看看他做了什么，PullMessageService继承了ServiceThread，并重写了他的run方法，我们来看看run方法中做了什么
java
@Override
public void run() {
logger.info(this.getServiceName() + " service started");
while (!this.isStopped()) {
try {
MessageRequest messageRequest = this.messageRequestQueue.take();
if (messageRequest.getMessageRequestMode() == MessageRequestMode.POP) {
//pop rocket5新特性，这里不做深入
this.popMessage((PopRequest) messageRequest);
} else {
//调用pullMessage方法获取消息
this.pullMessage((PullRequest) messageRequest);
}
} catch (InterruptedException ignored) {
} catch (Exception e) {
logger.error("Pull Message Service Run Method exception", e);
}
}
logger.info(this.getServiceName() + " service end");
}
private void pullMessage(final PullRequest pullRequest) {
final MQConsumerInner consumer = this.mQClientFactory.selectConsumer(pullRequest.getConsumerGroup());
if (consumer != null) {
DefaultMQPushConsumerImpl impl = (DefaultMQPushConsumerImpl) consumer;
impl.pullMessage(pullRequest);
} else {
logger.warn("No matched consumer for the PullRequest {}, drop it", pullRequest);
}
}在isStopped不为true的时候循环调用了DefaultMQPushConsumerImpl的#pullMessage方法
java
public void pullMessage(final PullRequest pullRequest) {
//获取队列
final ProcessQueue processQueue = pullRequest.getProcessQueue();
if (processQueue.isDropped()) {
log.info("the pull request[{}] is dropped.", pullRequest.toString());
return;
}
//设置最后一次pull的时间
pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis());
try {
this.makeSureStateOK();//确认服务状态
} catch (MQClientException e) {
log.warn("pullMessage exception, consumer state not ok", e);
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
return;
}
if (this.isPause()) {
log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup());
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND);
return;
}
//计算已经缓存的消息大小，如果容量超过可处理的大小，就later再pull
long cachedMessageCount = processQueue.getMsgCount().get();
long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024);
if (cachedMessageCount > this.defaultMQPushConsumer.getPullThresholdForQueue()) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL);
if ((queueFlowControlTimes++ % 1000) == 0) {
log.warn(
"the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}",
this.defaultMQPushConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);
}
return;
}
if (cachedMessageSizeInMiB > this.defaultMQPushConsumer.getPullThresholdSizeForQueue()) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL);
if ((queueFlowControlTimes++ % 1000) == 0) {
log.warn(
"the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}",
this.defaultMQPushConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);
}
return;
}
//是否顺序拉取
if (!this.consumeOrderly) {
//判断本地span和队列的span，如果队列的span比本地大，就later获取
if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL);
if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) {
log.warn(
"the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}",
processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(),
pullRequest, queueMaxSpanFlowControlTimes);
}
return;
}
} else {
//顺序拉取，判定队列是否能拿到锁
if (processQueue.isLocked()) {
if (!pullRequest.isPreviouslyLocked()) {
long offset = -1L;
try {
offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue());
if (offset < 0) {
throw new MQClientException(ResponseCode.SYSTEM_ERROR, "Unexpected offset " + offset);
}
} catch (Exception e) {
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
log.error("Failed to compute pull offset, pullResult: {}", pullRequest, e);
return;
}
boolean brokerBusy = offset < pullRequest.getNextOffset();
log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}",
pullRequest, offset, brokerBusy);
if (brokerBusy) {
log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}",
pullRequest, offset);
}
pullRequest.setPreviouslyLocked(true);
pullRequest.setNextOffset(offset);
}
} else {
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
log.info("pull message later because not locked in broker, {}", pullRequest);
return;
}
}
//获取队列和订阅信息
final MessageQueue messageQueue = pullRequest.getMessageQueue();
final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(messageQueue.getTopic());
if (null == subscriptionData) {
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
log.warn("find the consumer's subscription failed, {}", pullRequest);
return;
}
final long beginTimestamp = System.currentTimeMillis();
//创建回调对象
PullCallback pullCallback = new PullCallback() {
@Override
public void onSuccess(PullResult pullResult) {
if (pullResult != null) {
//处理消息
...... 省略
};
boolean commitOffsetEnable = false;
long commitOffsetValue = 0L;
if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) {
commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY);
if (commitOffsetValue > 0) {
commitOffsetEnable = true;
}
}
String subExpression = null;
boolean classFilter = false;
SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
if (sd != null) {
if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) {
subExpression = sd.getSubString();
}
classFilter = sd.isClassFilterMode();
}
int sysFlag = PullSysFlag.buildSysFlag(
commitOffsetEnable, // commitOffset
true, // suspend
subExpression != null, // subscription
classFilter // class filter
);
try {
this.pullAPIWrapper.pullKernelImpl(
pullRequest.getMessageQueue(),
subExpression,
subscriptionData.getExpressionType(),
subscriptionData.getSubVersion(),
pullRequest.getNextOffset(),
this.defaultMQPushConsumer.getPullBatchSize(),
this.defaultMQPushConsumer.getPullBatchSizeInBytes(),
sysFlag,
commitOffsetValue,
BROKER_SUSPEND_MAX_TIME_MILLIS,
CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND,
CommunicationMode.ASYNC,
pullCallback
);
} catch (Exception e) {
log.error("pullKernelImpl exception", e);
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
}
}在做了一系列校验后调用pullKernelImpl构建pull请求获取消息
java
public PullResult pullKernelImpl(
final MessageQueue mq,
final String subExpression,
final String expressionType,
final long subVersion,
final long offset,
final int maxNums,
final int maxSizeInBytes,
final int sysFlag,
final long commitOffset,
final long brokerSuspendMaxTimeMillis,
final long timeoutMillis,
final CommunicationMode communicationMode,
final PullCallback pullCallback
) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
FindBrokerResult findBrokerResult =
this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq),
this.recalculatePullFromWhichNode(mq), false);
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
findBrokerResult =
this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq),
this.recalculatePullFromWhichNode(mq), false);
}
if (findBrokerResult != null) {
{//校验版本
if (!ExpressionType.isTagType(expressionType)
&& findBrokerResult.getBrokerVersion() < MQVersion.Version.V4_1_0_SNAPSHOT.ordinal()) {
throw new MQClientException("The broker[" + mq.getBrokerName() + ", "
+ findBrokerResult.getBrokerVersion() + "] does not upgrade to support for filter message by " + expressionType, null);
}
}
int sysFlagInner = sysFlag;
if (findBrokerResult.isSlave()) {
sysFlagInner = PullSysFlag.clearCommitOffsetFlag(sysFlagInner);
}
//构建pull请求信息头
PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
requestHeader.setConsumerGroup(this.consumerGroup);
requestHeader.setTopic(mq.getTopic());
requestHeader.setQueueId(mq.getQueueId());
requestHeader.setQueueOffset(offset);
requestHeader.setMaxMsgNums(maxNums);
requestHeader.setSysFlag(sysFlagInner);
requestHeader.setCommitOffset(commitOffset);
requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis);
requestHeader.setSubscription(subExpression);
requestHeader.setSubVersion(subVersion);
requestHeader.setMaxMsgBytes(maxSizeInBytes);
requestHeader.setExpressionType(expressionType);
requestHeader.setBrokerName(mq.getBrokerName());
//获取topic所在的broker地址
String brokerAddr = findBrokerResult.getBrokerAddr();
if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) {
brokerAddr = computePullFromWhichFilterServer(mq.getTopic(), brokerAddr);
}
//向mq服务器发起pull请求
PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage(
brokerAddr,
requestHeader,
timeoutMillis,
communicationMode,
pullCallback);
return pullResult;
}
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}PullConsumer获取消息后如何处理
获取到消息后，会调用pullCallback的#onSuccess方法处理消息
java
@Override
public void onSuccess(PullResult pullResult) {
if (pullResult != null) {
//处理消息
pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult,
subscriptionData);
switch (pullResult.getPullStatus()) {
case FOUND:
long prevRequestOffset = pullRequest.getNextOffset();
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
long pullRT = System.currentTimeMillis() - beginTimestamp;
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullRT);
long firstMsgOffset = Long.MAX_VALUE;
if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
} else {
firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());
//扔到processQueue中去，具体的操作是更新本地的offset和缓存的消息数量，
boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
//提交消费任务
DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(
pullResult.getMsgFoundList(),
processQueue,
pullRequest.getMessageQueue(),
dispatchToConsume);
if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {
DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest,
DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());
} else {
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
}
}
if (pullResult.getNextBeginOffset() < prevRequestOffset
|| firstMsgOffset < prevRequestOffset) {
log.warn(
"[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}",
pullResult.getNextBeginOffset(),
firstMsgOffset,
prevRequestOffset);
}
break;
case NO_NEW_MSG:
case NO_MATCHED_MSG:
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
break;
case OFFSET_ILLEGAL:
//offset错误，提交一个线程去更新本地的队列信息，具体就是设置本地订阅的队列信息为Dropped,重新构建
log.warn("the pull request offset illegal, {} {}",
pullRequest.toString(), pullResult.toString());
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
pullRequest.getProcessQueue().setDropped(true);
DefaultMQPushConsumerImpl.this.executeTask(new Runnable() {
@Override
public void run() {
try {
DefaultMQPushConsumerImpl.this.offsetStore.updateAndFreezeOffset(pullRequest.getMessageQueue(),
pullRequest.getNextOffset());
DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue());
// removeProcessQueue will also remove offset to cancel the frozen status.
DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue());
DefaultMQPushConsumerImpl.this.rebalanceImpl.getmQClientFactory().rebalanceImmediately();
log.warn("fix the pull request offset, {}", pullRequest);
} catch (Throwable e) {
log.error("executeTaskLater Exception", e);
}
}
});
break;
default:
break;
}
}
}总结
- PullConsumer和PushConsumer都是通过rpc长轮询获取消息
- PushConsumer获取消息消费后再重新获取消息，吞吐效率会低于pull，消费力降低时，缺点是不会减少pullMessage的次数
- PullConsumer提供停止标识isStopped，可以通过makeStop配置