golang kafka sarama 源码解析

  • 消费者组重平衡

github.com/!shopify/[email protected]/consumer_group.go

go 复制代码
func (c *consumerGroup) newSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) {

	// 获取broker组协调器
	coordinator, err := c.client.Coordinator(c.groupID)
	if err != nil {
		if retries <= 0 {
			return nil, err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)
	}

	// 申请加入组
	// Join consumer group
	join, err := c.joinGroupRequest(coordinator, topics)
	if err != nil {
		_ = coordinator.Close()
		return nil, err
	}
	switch join.Err {
	case ErrNoError:
		c.memberID = join.MemberId
	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
		c.memberID = ""
		return c.newSession(ctx, topics, handler, retries)
	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
		if retries <= 0 {
			return nil, join.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)

	// 已经在重平衡期间
	case ErrRebalanceInProgress: // retry after backoff
		if retries <= 0 {
			return nil, join.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, false)
	default:
		return nil, join.Err
	}

	// 消费者组中的一个消费者作为leader,进行分区方案分配
	// Prepare distribution plan if we joined as the leader
	var plan BalanceStrategyPlan
	if join.LeaderId == join.MemberId {
		members, err := join.GetMembers()
		if err != nil {
			return nil, err
		}

		// 分配分区
		plan, err = c.balance(members)
		if err != nil {
			return nil, err
		}
	}

	// 同步给kafka,只有 leader会带上分区方案
	// Sync consumer group
	groupRequest, err := c.syncGroupRequest(coordinator, plan, join.GenerationId)
	if err != nil {
		_ = coordinator.Close()
		return nil, err
	}
	switch groupRequest.Err {
	case ErrNoError:
	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
		c.memberID = ""
		return c.newSession(ctx, topics, handler, retries)
	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
		if retries <= 0 {
			return nil, groupRequest.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, true)
	case ErrRebalanceInProgress: // retry after backoff
		if retries <= 0 {
			return nil, groupRequest.Err
		}

		return c.retryNewSession(ctx, topics, handler, retries, false)
	default:
		return nil, groupRequest.Err
	}

	// Retrieve and sort claims
	var claims map[string][]int32 // topic->partions
	// 如果有可消费的分区
	if len(groupRequest.MemberAssignment) > 0 {
		members, err := groupRequest.GetMemberAssignment()
		if err != nil {
			return nil, err
		}
		claims = members.Topics
		c.userData = members.UserData

		for _, partitions := range claims {
			sort.Sort(int32Slice(partitions))
		}
	}

	return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler)
}
  • 消费者拉取消息
go 复制代码
func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer {
	bc := &brokerConsumer{
		consumer:         c,
		broker:           broker,
		input:            make(chan *partitionConsumer),
		newSubscriptions: make(chan []*partitionConsumer),
		wait:             make(chan none),
		subscriptions:    make(map[*partitionConsumer]none),
		refs:             0,
	}

	go withRecover(bc.subscriptionManager)
	go withRecover(bc.subscriptionConsumer)

	return bc
}
相关推荐
吴_知遇1 小时前
【华为OD机试真题】428、连续字母长度 | 机试真题+思路参考+代码解析(E卷)(C++)
开发语言·c++·华为od
basketball6161 小时前
Python torchvision.transforms 下常用图像处理方法
开发语言·图像处理·python
宁酱醇1 小时前
各种各样的bug合集
开发语言·笔记·python·gitlab·bug
啊吧怪不啊吧2 小时前
Linux常见指令介绍下(入门级)
linux·开发语言·centos
谷晓光2 小时前
Python 中 `r` 前缀:字符串处理的“防转义利器”
开发语言·python
Tiger Z2 小时前
R 语言科研绘图第 41 期 --- 桑基图-基础
开发语言·r语言·贴图
chuxinweihui2 小时前
数据结构——二叉树,堆
c语言·开发语言·数据结构·学习·算法·链表
陈大大陈2 小时前
基于 C++ 的用户认证系统开发:从注册登录到Redis 缓存优化
java·linux·开发语言·数据结构·c++·算法·缓存
看到我,请让我去学习2 小时前
C语言基础(day0424)
c语言·开发语言·数据结构
studyer_domi2 小时前
Matlab 复合模糊PID
开发语言·matlab