golang kafka sarama 源码解析

  • 消费者组重平衡

github.com/!shopify/sarama@v1.27.2/consumer_group.go

go 复制代码
func (c *consumerGroup) newSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) {

	// 获取broker组协调器
	coordinator, err := c.client.Coordinator(c.groupID)
	if err != nil {
		if retries <= 0 {
			return nil, err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)
	}

	// 申请加入组
	// Join consumer group
	join, err := c.joinGroupRequest(coordinator, topics)
	if err != nil {
		_ = coordinator.Close()
		return nil, err
	}
	switch join.Err {
	case ErrNoError:
		c.memberID = join.MemberId
	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
		c.memberID = ""
		return c.newSession(ctx, topics, handler, retries)
	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
		if retries <= 0 {
			return nil, join.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)

	// 已经在重平衡期间
	case ErrRebalanceInProgress: // retry after backoff
		if retries <= 0 {
			return nil, join.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, false)
	default:
		return nil, join.Err
	}

	// 消费者组中的一个消费者作为leader,进行分区方案分配
	// Prepare distribution plan if we joined as the leader
	var plan BalanceStrategyPlan
	if join.LeaderId == join.MemberId {
		members, err := join.GetMembers()
		if err != nil {
			return nil, err
		}

		// 分配分区
		plan, err = c.balance(members)
		if err != nil {
			return nil, err
		}
	}

	// 同步给kafka,只有 leader会带上分区方案
	// Sync consumer group
	groupRequest, err := c.syncGroupRequest(coordinator, plan, join.GenerationId)
	if err != nil {
		_ = coordinator.Close()
		return nil, err
	}
	switch groupRequest.Err {
	case ErrNoError:
	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
		c.memberID = ""
		return c.newSession(ctx, topics, handler, retries)
	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
		if retries <= 0 {
			return nil, groupRequest.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)
	case ErrRebalanceInProgress: // retry after backoff
		if retries <= 0 {
			return nil, groupRequest.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, false)
	default:
		return nil, groupRequest.Err
	}

	// Retrieve and sort claims
	var claims map[string][]int32 // topic->partions
	// 如果有可消费的分区
	if len(groupRequest.MemberAssignment) > 0 {
		members, err := groupRequest.GetMemberAssignment()
		if err != nil {
			return nil, err
		}
		claims = members.Topics
		c.userData = members.UserData

		for _, partitions := range claims {
			sort.Sort(int32Slice(partitions))
		}
	}

	return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler)
}
  • 消费者拉取消息
go 复制代码
func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer {
	bc := &brokerConsumer{
		consumer:         c,
		broker:           broker,
		input:            make(chan *partitionConsumer),
		newSubscriptions: make(chan []*partitionConsumer),
		wait:             make(chan none),
		subscriptions:    make(map[*partitionConsumer]none),
		refs:             0,
	}

	go withRecover(bc.subscriptionManager)
	go withRecover(bc.subscriptionConsumer)

	return bc
}
相关推荐
----云烟----40 分钟前
QT中QString类的各种使用
开发语言·qt
KevinAha44 分钟前
Kafka 3.5 源码导读
kafka
求积分不加C44 分钟前
-bash: ./kafka-topics.sh: No such file or directory--解决方案
分布式·kafka
lsx20240644 分钟前
SQL SELECT 语句:基础与进阶应用
开发语言
nathan05291 小时前
javaer快速上手kafka
分布式·kafka
开心工作室_kaic1 小时前
ssm161基于web的资源共享平台的共享与开发+jsp(论文+源码)_kaic
java·开发语言·前端
向宇it1 小时前
【unity小技巧】unity 什么是反射?反射的作用?反射的使用场景?反射的缺点?常用的反射操作?反射常见示例
开发语言·游戏·unity·c#·游戏引擎
武子康1 小时前
Java-06 深入浅出 MyBatis - 一对一模型 SqlMapConfig 与 Mapper 详细讲解测试
java·开发语言·数据仓库·sql·mybatis·springboot·springcloud
转世成为计算机大神2 小时前
易考八股文之Java中的设计模式?
java·开发语言·设计模式
宅小海2 小时前
scala String
大数据·开发语言·scala