golang kafka sarama 源码解析

  • 消费者组重平衡

github.com/!shopify/[email protected]/consumer_group.go

go 复制代码
func (c *consumerGroup) newSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) {

	// 获取broker组协调器
	coordinator, err := c.client.Coordinator(c.groupID)
	if err != nil {
		if retries <= 0 {
			return nil, err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)
	}

	// 申请加入组
	// Join consumer group
	join, err := c.joinGroupRequest(coordinator, topics)
	if err != nil {
		_ = coordinator.Close()
		return nil, err
	}
	switch join.Err {
	case ErrNoError:
		c.memberID = join.MemberId
	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
		c.memberID = ""
		return c.newSession(ctx, topics, handler, retries)
	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
		if retries <= 0 {
			return nil, join.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)

	// 已经在重平衡期间
	case ErrRebalanceInProgress: // retry after backoff
		if retries <= 0 {
			return nil, join.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, false)
	default:
		return nil, join.Err
	}

	// 消费者组中的一个消费者作为leader,进行分区方案分配
	// Prepare distribution plan if we joined as the leader
	var plan BalanceStrategyPlan
	if join.LeaderId == join.MemberId {
		members, err := join.GetMembers()
		if err != nil {
			return nil, err
		}

		// 分配分区
		plan, err = c.balance(members)
		if err != nil {
			return nil, err
		}
	}

	// 同步给kafka,只有 leader会带上分区方案
	// Sync consumer group
	groupRequest, err := c.syncGroupRequest(coordinator, plan, join.GenerationId)
	if err != nil {
		_ = coordinator.Close()
		return nil, err
	}
	switch groupRequest.Err {
	case ErrNoError:
	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
		c.memberID = ""
		return c.newSession(ctx, topics, handler, retries)
	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
		if retries <= 0 {
			return nil, groupRequest.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)
	case ErrRebalanceInProgress: // retry after backoff
		if retries <= 0 {
			return nil, groupRequest.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, false)
	default:
		return nil, groupRequest.Err
	}

	// Retrieve and sort claims
	var claims map[string][]int32 // topic->partions
	// 如果有可消费的分区
	if len(groupRequest.MemberAssignment) > 0 {
		members, err := groupRequest.GetMemberAssignment()
		if err != nil {
			return nil, err
		}
		claims = members.Topics
		c.userData = members.UserData

		for _, partitions := range claims {
			sort.Sort(int32Slice(partitions))
		}
	}

	return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler)
}
  • 消费者拉取消息
go 复制代码
func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer {
	bc := &brokerConsumer{
		consumer:         c,
		broker:           broker,
		input:            make(chan *partitionConsumer),
		newSubscriptions: make(chan []*partitionConsumer),
		wait:             make(chan none),
		subscriptions:    make(map[*partitionConsumer]none),
		refs:             0,
	}

	go withRecover(bc.subscriptionManager)
	go withRecover(bc.subscriptionConsumer)

	return bc
}
相关推荐
lmryBC4916 分钟前
golang接口-interface
java·前端·golang
孤独得猿17 分钟前
Qt常用控件第一部分
服务器·开发语言·qt
慕斯策划一场流浪23 分钟前
fastGPT—nextjs—mongoose—团队管理之团队列表api接口实现
开发语言·前端·javascript·fastgpt env文件配置·fastgpt团队列表接口实现·fastgpt团队切换api·fastgpt团队切换逻辑
时光呢33 分钟前
JAVA常见的 JVM 参数及其典型默认值
java·开发语言·jvm
橙橙子23035 分钟前
c++柔性数组、友元、类模版
开发语言·c++·柔性数组
程序媛学姐42 分钟前
SpringKafka错误处理:重试机制与死信队列
java·开发语言·spring·kafka
2401_840192271 小时前
如何学习一门计算机技术
开发语言·git·python·devops
巷北夜未央1 小时前
Python每日一题(14)
开发语言·python·算法
浮尘笔记1 小时前
go-zero使用elasticsearch踩坑记:时间存储和展示问题
大数据·elasticsearch·golang·go
雾月552 小时前
LeetCode 914 卡牌分组
java·开发语言·算法·leetcode·职场和发展