【kafka】使用kafka client连接 kerberos认证的 kafka,scala版

注意keytab路径中不要使用\\,都使用/作为分隔符

使用kerberos需要配置jaas如下日志打印,两个配置至少设置一个:

复制代码
[DEBUG] org.apache.kafka.common.security.JaasContext:106 --- System property 'java.security.auth.login.config' and Kafka SASL property 'sasl.jaas.config' are not set, using default JAAS configuration.
scala 复制代码
import org.slf4j.Logger
trait Logging {
  val LOG: Logger = org.slf4j.LoggerFactory.getLogger(this.getClass)
}
scala 复制代码
import org.apache.kafka.clients.producer.ProducerRecord
import org.scalatest.FunSuite
import java.util.Properties
import org.apache.kafka.clients.CommonClientConfigs
import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.config.SaslConfigs
import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
import java.time.Duration
import scala.collection.JavaConversions._

class KafkaClientTest extends FunSuite with Logging {

  val JAAS_CONFIG_KEYTAB_TEMPLATE: String =
    s"""
       |com.sun.security.auth.module.Krb5LoginModule required
       |debug=true
       |doNotPrompt=true
       |storeKey=true
       |useKeyTab=true
       |keyTab="%s"
       |principal="%s";
       |""".stripMargin

  val topic = "tmp_test"
  val bootstrapServers = "01.xxx.com:6667,01.xxx..com:6667,01.xxx..com:6667"

  val principal = "xxx@XXXXX.COM"
  val keytab = "D:/xxx/xxx.keytab"
  val krb5conf = "D:/xxx/krb5.conf"

  def getProducerProps: Properties = {
    val properties = new Properties()
    properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers)
    properties.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT")
    properties.setProperty(SaslConfigs.SASL_MECHANISM, "GSSAPI")
    properties.setProperty(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka")
    properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)
    properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)
    properties
  }

  def getConsumerProps: Properties = {
    val properties = new Properties()
    properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers)
    properties.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT")
    properties.setProperty(SaslConfigs.SASL_MECHANISM, "GSSAPI")
    properties.setProperty(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka")
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer].getName)
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer].getName)
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "tester")
    properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
    properties
  }

  def setEnv(props: Properties) = {
    System.setProperty("java.security.krb5.conf", krb5conf)

    // 以下二者选其中之一就可以了。
    // 方式一:
    System.setProperty("java.security.auth.login.config", "D:/configs/kafka_client_jaas.conf")

    // 方式二:
    val jaasStr = JAAS_CONFIG_KEYTAB_TEMPLATE.format(keytab, principal).trim
    LOG.warn(s"format str: \n${jaasStr}")
    props.setProperty(SaslConfigs.SASL_JAAS_CONFIG, jaasStr)
  }

  test("produce") {
    val props = getConsumerProps
    setEnv(props)
    try {
      val producer = new KafkaProducer[String, String](props)
      var counter = 0;
      while (true) {
        val record = new ProducerRecord[String, String](topic, s"hello ${counter}")
        val fu = producer.send(record)
        Thread.sleep(3000L)
        if (counter % 10 == 0) {
          producer.flush()
        }
        counter += 1
      }
    } catch {
      case e: Exception =>
        throw new RuntimeException(e)
    }
  }

  test("consumer") {
    val props = getConsumerProps
    setEnv(props)
    val consumer = new KafkaConsumer(props)
    consumer.subscribe(List(topic))
    while (true) {
      val record = consumer.poll(Duration.ofSeconds(3))
      val it = record.iterator()
      while (it.hasNext) {
        LOG.info(s"${it.next().value()}")
      }
    }
  }
}

kafka_client_jaas.conf 文件内容:

文件模板参考:KAFKA_HOME/conf/kafka_client_jaas.conf

conf 复制代码
KafkaClient {
com.sun.security.auth.module.Krb5LoginModule required
    useKeyTab = true
    useTicketCache=false
    storeKey = true
    keyTab="D:/***/xxx.keytab"
    principal="xxx@XXXXX.COM"
    serviceName="kafka";
};
相关推荐
开开心心就好39 分钟前
电脑息屏工具,一键黑屏超方便
开发语言·javascript·电脑·scala·erlang·perl
{⌐■_■}11 小时前
【Kafka】登录日志处理的三次阶梯式优化实践:从同步写入到Kafka多分区批处理
数据库·分布式·mysql·kafka·go
CodeWithMe11 小时前
【Note】《Kafka: The Definitive Guide》 第九章:Kafka 管理与运维实战
运维·分布式·kafka
sql2008help11 小时前
1-Kafka介绍及常见应用场景
分布式·kafka
vivo互联网技术1 天前
vivo Pulsar 万亿级消息处理实践(3)-KoP指标异常修复
java·大数据·服务器·后端·kafka·消息队列·pulsar
Haoea!1 天前
Flink-05学习 接上节,将FlinkJedisPoolConfig 从Kafka写入Redis
学习·flink·kafka
搬砖天才、2 天前
kafka集群安装
分布式·kafka
即将雄起的运维玩家2 天前
kafka-kraft+SASL版本升级3.7.0-->3.9.1
分布式·kafka