【kafka】使用kafka client连接 kerberos认证的 kafka,scala版

注意keytab路径中不要使用\\,都使用/作为分隔符

使用kerberos需要配置jaas如下日志打印,两个配置至少设置一个:

复制代码
[DEBUG] org.apache.kafka.common.security.JaasContext:106 --- System property 'java.security.auth.login.config' and Kafka SASL property 'sasl.jaas.config' are not set, using default JAAS configuration.
scala 复制代码
import org.slf4j.Logger
trait Logging {
  val LOG: Logger = org.slf4j.LoggerFactory.getLogger(this.getClass)
}
scala 复制代码
import org.apache.kafka.clients.producer.ProducerRecord
import org.scalatest.FunSuite
import java.util.Properties
import org.apache.kafka.clients.CommonClientConfigs
import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.config.SaslConfigs
import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
import java.time.Duration
import scala.collection.JavaConversions._

class KafkaClientTest extends FunSuite with Logging {

  val JAAS_CONFIG_KEYTAB_TEMPLATE: String =
    s"""
       |com.sun.security.auth.module.Krb5LoginModule required
       |debug=true
       |doNotPrompt=true
       |storeKey=true
       |useKeyTab=true
       |keyTab="%s"
       |principal="%s";
       |""".stripMargin

  val topic = "tmp_test"
  val bootstrapServers = "01.xxx.com:6667,01.xxx..com:6667,01.xxx..com:6667"

  val principal = "xxx@XXXXX.COM"
  val keytab = "D:/xxx/xxx.keytab"
  val krb5conf = "D:/xxx/krb5.conf"

  def getProducerProps: Properties = {
    val properties = new Properties()
    properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers)
    properties.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT")
    properties.setProperty(SaslConfigs.SASL_MECHANISM, "GSSAPI")
    properties.setProperty(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka")
    properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)
    properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)
    properties
  }

  def getConsumerProps: Properties = {
    val properties = new Properties()
    properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers)
    properties.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT")
    properties.setProperty(SaslConfigs.SASL_MECHANISM, "GSSAPI")
    properties.setProperty(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka")
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer].getName)
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer].getName)
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "tester")
    properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
    properties
  }

  def setEnv(props: Properties) = {
    System.setProperty("java.security.krb5.conf", krb5conf)

    // 以下二者选其中之一就可以了。
    // 方式一:
    System.setProperty("java.security.auth.login.config", "D:/configs/kafka_client_jaas.conf")

    // 方式二:
    val jaasStr = JAAS_CONFIG_KEYTAB_TEMPLATE.format(keytab, principal).trim
    LOG.warn(s"format str: \n${jaasStr}")
    props.setProperty(SaslConfigs.SASL_JAAS_CONFIG, jaasStr)
  }

  test("produce") {
    val props = getConsumerProps
    setEnv(props)
    try {
      val producer = new KafkaProducer[String, String](props)
      var counter = 0;
      while (true) {
        val record = new ProducerRecord[String, String](topic, s"hello ${counter}")
        val fu = producer.send(record)
        Thread.sleep(3000L)
        if (counter % 10 == 0) {
          producer.flush()
        }
        counter += 1
      }
    } catch {
      case e: Exception =>
        throw new RuntimeException(e)
    }
  }

  test("consumer") {
    val props = getConsumerProps
    setEnv(props)
    val consumer = new KafkaConsumer(props)
    consumer.subscribe(List(topic))
    while (true) {
      val record = consumer.poll(Duration.ofSeconds(3))
      val it = record.iterator()
      while (it.hasNext) {
        LOG.info(s"${it.next().value()}")
      }
    }
  }
}

kafka_client_jaas.conf 文件内容:

文件模板参考:KAFKA_HOME/conf/kafka_client_jaas.conf

conf 复制代码
KafkaClient {
com.sun.security.auth.module.Krb5LoginModule required
    useKeyTab = true
    useTicketCache=false
    storeKey = true
    keyTab="D:/***/xxx.keytab"
    principal="xxx@XXXXX.COM"
    serviceName="kafka";
};
相关推荐
亲爱的非洲野猪1 小时前
Kafka消息积压的多维度解决方案:超越简单扩容的完整策略
java·分布式·中间件·kafka
亲爱的非洲野猪7 小时前
Kafka “假死“现象深度解析与解决方案
分布式·kafka
CodeWithMe7 小时前
【Note】《Kafka: The Definitive Guide》第三章: Kafka 生产者深入解析:如何高效写入 Kafka 消息队列
分布式·kafka
虾条_花吹雪7 小时前
2、Connecting to Kafka
分布式·ai·kafka
亲爱的非洲野猪11 小时前
Kafka消息积压全面解决方案:从应急处理到系统优化
分布式·kafka
prince051 天前
Kafka 生产者和消费者高级用法
分布式·kafka·linq
菜萝卜子1 天前
【Project】基于kafka的高可用分布式日志监控与告警系统
分布式·kafka
csdn_aspnet1 天前
在 Windows 上安装和运行 Apache Kafka
windows·kafka
真实的菜2 天前
Kafka生态整合深度解析:构建现代化数据架构的核心枢纽
架构·kafka·linq
茫茫人海一粒沙3 天前
理解 Confluent Schema Registry:Kafka 生态中的结构化数据守护者
分布式·kafka