进入
DoubleCloud
创建一个kafka
1 选择语言
2 运行curl 的url命令启动一个topic
3 生成对应语言的token
4 复制3中的配置文件到本地，命名为client.properties
5 复制客户端代码
对python和java客户端代码进行了重写，java改成了kotlin：
配置文件
# Required connection configs for Kafka producer, consumer, and admin
bootstrap.servers=
security.protocol=SASL_SSL
sasl.mechanisms=PLAIN
sasl.username=
sasl.password=
group.id=
auto.offset.reset=earliest
# Best practice for higher availability in librdkafka clients prior to 1.7
session.timeout.ms=45000
import time
from confluent_kafka import Producer, Consumer
import asyncio
import threading
class KafkaClient:
def __init__(self, config_file):
self.config = self.read_config(config_file)
def read_config(self, config_file):
config = {}
with open(config_file) as fh:
for line in fh:
line = line.strip()
if len(line) != 0 and line[0] != "#":
parameter, value = line.strip().split('=', 1)
config[parameter] = value.strip()
return config
def produce(self, topic, key, value):
# Creates a new producer instance
producer = Producer(self.config)
# Produces a sample message
producer.produce(topic, key=key, value=value)
print(f"Produced message to topic {topic}: key = {key:12} value = {value:12}")
# Send any outstanding or buffered messages to the Kafka broker
producer.flush()
def consume_async(self, topic, callback=None, group_id="python-group-1", auto_offset_reset="earliest"):
# Sets the consumer group ID and offset
self.config["group.id"] = group_id
self.config["auto.offset.reset"] = auto_offset_reset
consumer = Consumer(self.config)
consumer.subscribe([topic])
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
if callback is not None:
loop.run_until_complete(callback(consumer))
def consume(self, topic, callback=None):
thread = threading.Thread(target=self.consume_async, args=(topic, callback,))
thread.start()
return thread
async def consume_async(consumer):
try:
while True:
msg = consumer.poll(1.0)
if msg is not None:
break
if not msg.error():
key = msg.key().decode("utf-8")
value = msg.value().decode("utf-8")
print(f"Consumed message: key = {key:12} value = {value:12}")
except KeyboardInterrupt:
pass
finally:
consumer.close()
config_file_path = ".\\client.properties"
topic = "test"
key = "key"
value = "value"
kafka_client = KafkaClient(config_file_path)
kafka_client.produce(topic, key, value)
thread = kafka_client.consume(topic, consume_async)
配置文件
# Required connection configs for Kafka producer, consumer, and admin
bootstrap.servers=
security.protocol=SASL_SSL
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='GHFXZDIOMQW3IPKA' password='TimUk7hj/EwTiB031lA95LeKfXN3t2Ddnw+izhKx3+7wFxZKMLGEqTOnneTKrlQQ';
sasl.mechanism=PLAIN
# Required for correctness in Apache Kafka clients prior to 2.6
client.dns.lookup=use_all_dns_ips
key.serializer=org.apache.kafka.common.serialization.StringSerializer
value.serializer=org.apache.kafka.common.serialization.StringSerializer
# Best practice for higher availability in Apache Kafka clients prior to 3.0
session.timeout.ms=45000
topic=
group.id=
auto.offset.reset=earliest
key.deserializer=org.apache.kafka.common.serialization.StringDeserializer
value.deserializer=org.apache.kafka.common.serialization.StringDeserializer
# Best practice for Kafka producer to prevent data loss
acks=all
java(kotiln)
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.newFixedThreadPoolContext
import org.apache.kafka.clients.consumer.ConsumerRecords
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.clients.producer.ProducerRecord
import java.io.Closeable
import java.io.FileInputStream
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Paths
import java.time.Duration
import java.util.*
class KafkaClient<T, V> : Closeable {
private var producer: KafkaProducer<T, V>? = null
private var fileConfig: Properties? = null
val TOPIC = "topic"
private val DURATION = 100L
private val POOLSIZE = 10
private val DISPATCHER = newFixedThreadPoolContext(POOLSIZE, "CoroutinePool")
private val SCOPE = CoroutineScope(DISPATCHER)
constructor(configPath: String? = null, config: Properties? = null) {
if (config == null && configPath == null) throw Exception("don't have any config")
var config1 = Properties()
if (configPath != null) {
fileConfig = readConfig(configPath)
fileConfig?.let { config1.putAll(it) }
}
if (config != null) {
config1.putAll(config)
}
producer = KafkaProducer(config1)
}
fun produce(key: T, value: V, topic: String? = null) {
producer?.send(ProducerRecord(topic ?: (fileConfig?.getProperty(TOPIC) as String), key, value))
}
fun consume(func: suspend (ConsumerRecords<T, V>) -> Unit) {
val consumer: KafkaConsumer<T, V> = KafkaConsumer(fileConfig)
consumer.subscribe(Arrays.asList(fileConfig?.getProperty(TOPIC)))
SCOPE.launch {
while (true) {
val records: ConsumerRecords<T, V> = consumer.poll(Duration.ofMillis(DURATION))
func(records)
delay(DURATION)
}
}
}
@Throws(IOException::class)
fun readConfig(configFile: String): Properties {
if (!Files.exists(Paths.get(configFile))) {
throw IOException("$configFile not found.")
}
val config = Properties()
FileInputStream(configFile).use { inputStream -> config.load(inputStream) }
return config
}
override fun close() {
producer?.close()
}
}
fun main() {
val cli =
KafkaClient<String, String>("D:\\src\\main\\java\\com\\tr\\robot\\io\\kafka\\client.properties")
cli.consume {
println("test beg")
for (record in it) {
println(
String.format(
"Consumed message from topic %s: key = %s value = %s", cli.TOPIC, record.key(), record.value()
)
)
}
println("test end")
}
// Give some time for the consumer to start
Thread.sleep(2000)
cli.produce("key1", "test")
// Give some time for the consumer to consume the message
Thread.sleep(5000)
}