doris+kafka安装部署(单机+集群)六安装部署

powershell 复制代码
# 分步实施计划
3个Follower(3台各1个FE-Follower);3台全部部署BE;3台机器每台部署1个BE的Broker用于读取hdfs hive等;
同时doris部署用doris用户,密码为 DorisPassw0rd;同时kafka部署用kafka用户,密码为 KafkaPassw0rd;
思路:
1. node1(172.18.206.64)把4个压缩包分发到node2、node3的`/opt/software`
2. 三台机器清理旧JAVA_HOME环境变量,统一部署 JDK17
3. 先单节点测试:只在node1部署Doris(1FE+1BE)、单节点Kafka,用mysql客户端连通测试
4. node1全部验证正常后,再扩容成Doris 1 Master+2 Observer FE、3BE集群;再扩容Kafka成KRaft三节点集群

FE:3Follower(1Leader+2Follower,可自动选主,不用Observer)
BE:3台
Doris Broker:3台(专门对接HDFS/Hive)
Kafka:3节点KRaft集群


# 步骤1 三台机器创建操作系统用户(三台全部执行)
# 创建doris用户
useradd doris
echo "DorisPassw0rd" | passwd --stdin doris

# 创建kafka用户
useradd kafka
echo "KafkaPassw0rd" | passwd --stdin kafka

# 目录权限
mkdir -p /opt/software /data/ssd/doris/fe /data/ssd/doris_be /data/hdd/doris_be /data/hdd/kafka

# 属主
chown -R doris:doris /data/ssd/doris
chown -R doris:doris /data/ssd/doris_be
chown -R doris:doris /data/hdd/doris_be
chown -R kafka:kafka /data/hdd/kafka

# 父目录必须给执行权限,否则子目录再700也访问不了
chmod 755 /data
chmod 755 /data/ssd
chmod 755 /data/hdd

# 业务数据目录收紧仅属主读写
chmod 700 /data/ssd/doris/fe
chmod 700 /data/ssd/doris_be
chmod 700 /data/hdd/doris_be
chmod 700 /data/hdd/kafka

# 安装包目录
chmod 755 /opt/software

# 验证
su - doris
ls /data/ssd/doris_be
ls /data/ssd/doris/fe
ls /data/hdd/doris_be

su - kafka
ls /data/hdd/kafka



# 步骤2 node1 root分发4个压缩包到node2、node3 /opt/software
# ssh免密快速配置(本机执行)
ssh-keygen -t ecdsa -N ''
ssh-copy-id root@172.18.206.65
ssh-copy-id root@172.18.206.66
ssh-copy-id root@172.18.206.64


cd /opt/software

scp apache-doris-4.0.8-bin-x64.tar.gz root@172.18.206.65:/opt/software/
scp kafka_2.13-3.7.0.tgz root@172.18.206.65:/opt/software/
scp mysql-8.0.36-linux-glibc2.12-x86_64.tar.xz root@172.18.206.65:/opt/software/
scp OpenJDK17U-jdk_x64_linux_hotspot_17.0.20.1_1.tar.gz root@172.18.206.65:/opt/software/
scp numactl-2.0.12-5.el7.x86_64.rpm root@172.18.206.65:/opt/software/
scp numactl-libs-2.0.12-5.el7.x86_64.rpm root@172.18.206.65:/opt/software/

scp apache-doris-4.0.8-bin-x64.tar.gz root@172.18.206.66:/opt/software/
scp kafka_2.13-3.7.0.tgz root@172.18.206.66:/opt/software/
scp mysql-8.0.36-linux-glibc2.12-x86_64.tar.xz root@172.18.206.66:/opt/software/
scp OpenJDK17U-jdk_x64_linux_hotspot_17.0.20.1_1.tar.gz root@172.18.206.66:/opt/software/
scp numactl-2.0.12-5.el7.x86_64.rpm root@172.18.206.66:/opt/software/
scp numactl-libs-2.0.12-5.el7.x86_64.rpm root@172.18.206.66:/opt/software/

# 传输完成后校验md5(必做,防止包损坏)
# node2、node3校验包
ls -lh /opt/software
md5sum apache-doris-4.0.8-bin-x64.tar.gz
md5sum kafka_2.13-3.7.0.tgz
md5sum mysql-8.0.36-linux-glibc2.12-x86_64.tar.xz
md5sum OpenJDK17U-jdk_x64_linux_hotspot_17.0.20.1_1.tar.gz



# 步骤 3 三台机器清理旧JAVA_HOME,部署JDK17(三台root执行)
cd /opt/software
tar -zxvf OpenJDK17U-jdk_x64_linux_hotspot_17.0.20.1_1.tar.gz

sed -i '/^export JAVA_HOME/d' /etc/profile
sed -i '/JAVA_HOME.*bin/d' /etc/profile

cat >> /etc/profile <<'EOF'
export JAVA_HOME=/opt/software/jdk-17.0.20.1+1
export PATH=$JAVA_HOME/bin:$PATH
EOF

source /etc/profile
java -version
echo $JAVA_HOME
ls $JAVA_HOME/bin/java

### 步骤 4 node1单节点测试
#### 4.1 node1解压doris,授权doris用户
cd /opt/software
tar -zxvf apache-doris-4.0.8-bin-x64.tar.gz
chown -R doris:doris apache-doris-4.0.8-bin-x64
su - doris
cd /opt/software/apache-doris-4.0.8-bin-x64/

# 验证
ls /data/ssd/doris_be
ls /data/ssd/doris/fe
ls /data/hdd/doris_be

# fe/conf/fe.conf
meta_dir = /data/ssd/doris/fe
priority_networks=172.18.206.64/24
http_port = 8030
query_port = 9030

# be/conf/be.conf  doris配置文件要求`key=value`不能有空格!!!多路径:`路径1,medium=SSD;路径2,medium=HDD`分号`;`分割不同磁盘；逗号`,`分割同一路径的属性。
priority_networks=172.18.206.64/24
# Doris BE mem_limit通常设置为物理内存的70%~80%
mem_limit=300G 
storage_root_path=/data/ssd/doris_be,medium:SSD;/data/hdd/doris_be,medium:HDD
cold_data_migrate_max_io_mb_per_sec=50

# 4.2 doris用户启动FE
cd fe
./bin/start_fe.sh --daemon
tail -f log/fe.log
# 等待日志`successfully finished`

# 4.3 root 设置doris root密码
cd /opt/software
tar -Jxf mysql-8.0.36-linux-glibc2.12-x86_64.tar.xz
#重命名
mv /opt/software/mysql-8.0.36-linux-glibc2.12-x86_64 /opt/software/mysql8

#清理旧的错误行
sed -i '/mysql8\/bin/d' /etc/profile
echo 'export PATH=$PATH:/opt/software/mysql8/bin' >> /etc/profile
source /etc/profile
#验证,能输出版本就成功
mysql --version

mysql -h127.0.0.1 -P9030 -uroot -e "SET PASSWORD FOR 'root'@'%' = PASSWORD('DorisPassw0rd');"


cd ../be
numactl --interleave=all ./bin/start_be.sh --daemon
# 正确查看日志
tail -f /opt/software/apache-doris-4.0.8-bin-x64/be/log/be.INFO
# 看启动输出(stdout)
tail -f /opt/software/apache-doris-4.0.8-bin-x64/be/log/be.out
#等待be启动完成

#4.4 添加本机BE,doris用户启动BE BE启动完毕,注册到FE,二选一,推荐SQL方式 只能执行一次!!!
#方式1 curl http接口
curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_backend?host=172.18.206.64&port=9050"
#方式2 mysql客户端执行SQL(更推荐)
mysql -h127.0.0.1 -P9030 -uroot -pDorisPassw0rd 
ALTER SYSTEM ADD BACKEND '172.18.206.64:9050';
SHOW BACKENDS; --查看状态

# web 访问`http://172.18.206.64:8030`确认FE、BE Alive


# 4.5 node1 kafka用户部署单节点KRaft kafka
exit
cd /opt/software
tar -zxvf kafka_2.13-3.7.0.tgz
chown -R kafka:kafka kafka_2.13-3.7.0
cd /opt/software/kafka_2.13-3.7.0
cp config/server.properties config/server.properties.bak

# 验证
ls /data/hdd/kafka

# config/server.properties
process.roles=broker,controller
node.id=1
broker.id=1
controller.quorum.voters=1@172.18.206.64:9093
listeners=PLAINTEXT://:9092,CONTROLLER://:9093
inter.broker.listener.name=PLAINTEXT
advertised.listeners=PLAINTEXT://172.18.206.64:9092
controller.listener.names=CONTROLLER
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
log.dirs=/data/hdd/kafka


su - kafka
cd /opt/software/kafka_2.13-3.7.0
bin/kafka-storage.sh format -t $(bin/kafka-storage.sh random-uuid) -c config/server.properties
# 成功标志:`Formatting /data/hdd/kafka with cluster id: xxxxx`
bin/kafka-server-start.sh -daemon config/server.properties
bin/kafka-broker-api-versions.sh --bootstrap-server 172.18.206.64:9092
# 查看进程是否存在
ps -ef | grep kafka-server

# 4.6 mysql客户端测试doris
exit
cd /opt/software
tar -Jxvf mysql-8.0.36-linux-glibc2.12-x86_64.tar.xz
ln -s /opt/software/mysql-8.0.36-linux-glibc2.12-x86_64/bin/mysql /usr/local/bin/mysql

mysql -h172.18.206.64 -P9030 -uroot -pDorisPassw0rd

# 测试SQL
show databases;
create database test;
use test;
create table t1(id int) duplicate key(id) distributed by hash(id) buckets 1 PROPERTIES ("replication_num" = "1");
insert into t1 values(1);
select * from t1;

# 4.7 测试Kafka,Kafka数据落Doris(node1执行)
# 4.7.1 kafka用户创建测试topic
su - kafka
cd /opt/software/kafka_2.13-3.7.0
bin/kafka-topics.sh --create --topic doris_test_topic --bootstrap-server 172.18.206.64:9092 --partitions 3 --replication-factor 1
# 查看topic详情
bin/kafka-topics.sh --describe --topic doris_test_topic --bootstrap-server 172.18.206.64:9092
#查看topic
bin/kafka-topics.sh --list --bootstrap-server 172.18.206.64:9092

# 新开终端
# 生产者,输入内容回车发送
cd /opt/software/kafka_2.13-3.7.0
bin/kafka-console-producer.sh --bootstrap-server 172.18.206.64:9092 --topic doris_test_topic
# 再开一个终端消费:
cd /opt/software/kafka_2.13-3.7.0
bin/kafka-console-consumer.sh --bootstrap-server 172.18.206.64:9092 --topic doris_test_topic --from-beginning
# ✅生产者发消息,消费者能收到,说明kafka读写正常。

# kafka节点执行删除topic
bin/kafka-topics.sh --delete --topic doris_test_topic --bootstrap-server 172.18.206.64:9092
# 重建topic
bin/kafka-topics.sh --create --topic doris_test_topic --bootstrap-server 172.18.206.64:9092 --partitions 3 --replication-factor 1


# 4.7.2 Doris建库、建接收数据的表(mysql客户端执行)
mysql -h172.18.206.64 -P9030 -uroot -pDorisPassw0rd


create database kafka2doris;
use kafka2doris;

CREATE TABLE kafka_sink_table (
    id INT,
    name STRING,
    ts BIGINT
)
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 3
PROPERTIES(
    "replication_num" = "1"
);


# 4.7.3 创建RoutineLoad任务消费kafka写入doris
# 需要删除的话
# STOP ROUTINE LOAD FOR test_routine_load;
CREATE ROUTINE LOAD kafka2doris.test_routine_load ON kafka_sink_table
COLUMNS(id,name,ts)
PROPERTIES
(
    "desired_concurrent_number"="3",
    "max_batch_interval" = "10",
    "max_batch_rows" = "200000",
    "max_batch_size" = "104857600",
    "format" = "json",
    "jsonpaths" = "[\"$.id\",\"$.name\",\"$.ts\"]"
    "max_error_number" = "100" --允许100条错误
)
FROM KAFKA
(
    "kafka_broker_list" = "172.18.206.64:9092",
    "kafka_topic" = "doris_test_topic",
    "kafka_partitions" = "0,1,2",
    "kafka_offsets" = "OFFSET_BEGINNING,OFFSET_BEGINNING,OFFSET_BEGINNING"
);

-- 查看routine load任务状态  指定任务名(推荐)
SHOW ROUTINE LOAD FOR test_routine_load;
-- 只看运行中的任务
SHOW ROUTINE LOAD FROM kafka2doris;
-- 查看全部,包含已停止/历史任务
SHOW ROUTINE LOAD FROM kafka2doris WHERE IncludeHistory = true;


#### 4.7.4 kafka生产测试json数据
# 新开终端,kafka用户执行生产者
su - kafka
cd /opt/software/kafka_2.13-3.7.0
bin/kafka-console-producer.sh --broker-list 172.18.206.64:9092 --topic doris_test_topic
# 输入测试json消息,逐条发送:
{"id":1,"name":"zhangsan","ts":1750000000000}
{"id":2,"name":"lisi","ts":1750000001000}
{"id":100,"name":"wangwu","ts":1750000002000}

#### 4.7.5 Doris验证数据落地
use kafka2doris;
select * from kafka_sink_table;
-- 查看routine load运行指标
SHOW ROUTINE LOAD FOR test_routine_load\G
-- 暂停任务
PAUSE ROUTINE LOAD FOR test_routine_load;
-- 恢复任务
RESUME ROUTINE LOAD FOR test_routine_load;
-- 删除任务
DROP ROUTINE LOAD FOR test_routine_load;


# 验证通过标志:
# 1. RoutineLoad状态为`RUNNING`
# 2. 消费偏移量持续上涨
# 3. select查询能看到kafka发送过来的3条测试数据

# 从头消费全部消息
bin/kafka-console-consumer.sh \
--bootstrap-server 172.18.206.64:9092,172.18.206.65:9092,172.18.206.66:9092 \
--topic doris_test_topic \
--from-beginning
# 只看最新的消息
bin/kafka-console-consumer.sh \
--bootstrap-server 172.18.206.64:9092,172.18.206.65:9092,172.18.206.66:9092 \
--topic doris_test_topic
# 或者看元数据
cd /data/hdd/kafka
ls

/opt/software/kafka_2.13-3.7.0/bin/kafka-dump-log.sh \
--files /data/hdd/kafka/doris_test_topic-0/00000000000000000000.log \
--print-data-log \
--deep-iteration







> ✅单节点全部验证通过,再扩容
### 步骤 5 扩容Doris集群:3FE-Follower +3BE +3Doris-Broker
# node2、node3 root解压doris包,授权doris用户,修改fe.conf/be.conf为本机IP,**暂不启动进程**
cd /opt/software
tar -zxvf apache-doris-4.0.8-bin-x64.tar.gz
chown -R doris:doris apache-doris-4.0.8-bin-x64

# node2 /opt/software/apache-doris-4.0.8-bin-x64/fe/conf/fe.conf
meta_dir = /data/ssd/doris/fe
priority_networks=172.18.206.65/24
http_port = 8030
query_port = 9030
# node3 /opt/software/apache-doris-4.0.8-bin-x64/fe/conf/fe.conf
meta_dir = /data/ssd/doris/fe
priority_networks=172.18.206.66/24
http_port = 8030
query_port = 9030
# node2 /opt/software/apache-doris-4.0.8-bin-x64/be/conf/be.conf
priority_networks=172.18.206.65/24
mem_limit=300G
storage_root_path=/data/ssd/doris_be,medium:SSD;/data/hdd/doris_be,medium:HDD
cold_data_migrate_max_io_mb_per_sec=50
# node3 /opt/software/apache-doris-4.0.8-bin-x64/be/conf/be.conf
priority_networks=172.18.206.66/24
mem_limit=300G
storage_root_path=/data/ssd/doris_be,medium:SSD;/data/hdd/doris_be,medium:HDD
cold_data_migrate_max_io_mb_per_sec=50

# 5.1 node1执行添加2个FE-Follower
# 建议用sql方式
mysql -h172.18.206.64 -P9030 -uroot -pDorisPassw0rd
ALTER SYSTEM ADD FOLLOWER "172.18.206.65:9010";
ALTER SYSTEM ADD FOLLOWER "172.18.206.66:9010";

curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_follower?host=172.18.206.65&port=9010"
curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_follower?host=172.18.206.66&port=9010"


# 5.2 node2、node3 doris用户启动FE
su - doris
cd /opt/software/apache-doris-4.0.8-bin-x64/fe
./bin/start_fe.sh --helper 172.18.206.64:9010 --daemon
# `--helper` 只需要**第一次启动的时候加**；后续重启,直接 `./bin/start_fe.sh --daemon` 即可,不需要再带helper。

# 如果之前直接 ./bin/start_fe.sh --daemon 了,需要停止fe,清空meta目录下bdb、image,重新用--helper启动
su - doris
cd /opt/software/apache-doris-4.0.8-bin-x64/fe
./bin/stop_fe.sh --daemon
# 进入meta目录
cd /data/ssd/doris/fe
# 删除bdb、image目录,这两个是元数据核心
rm -rf bdb image
# 只删bdb、image,不要删整个meta_dir目录,目录本身保留。
# 清空之后,本地FE没有任何旧集群元数据,才能重新从helper拉取。
tail -f log/fe.log

# webUI确认:1Leader,2Follower

# 5.3 node2、node3 doris用户启动BE
su - doris
cd /opt/software/apache-doris-4.0.8-bin-x64/be
numactl --interleave=all ./bin/start_be.sh --daemon
# 如果之前加入过独立FE,需要清空BE存储目录下元数据(重点！清除旧cluster id) conf文件里面写的storage_root_path=/data/ssd/doris_be,/data/hdd/doris_be
su - doris
cd /opt/software/apache-doris-4.0.8-bin-x64/be
./bin/stop_be.sh
# node2执行
rm -rf /data/ssd/doris_be/*
rm -rf /data/hdd/doris_be/*
# node3同样执行
rm -rf /data/ssd/doris_be/*
rm -rf /data/hdd/doris_be/*

#### 5.4 node1添加node2、node3BE
# 建议用sql方式
mysql -h172.18.206.64 -P9030 -uroot -pDorisPassw0rd
ALTER SYSTEM ADD BACKEND "172.18.206.65:9050";
ALTER SYSTEM ADD BACKEND "172.18.206.66:9050";

curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_backend?host=172.18.206.65&port=9050"
curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_backend?host=172.18.206.66&port=9050"


-- 查看FE
show frontends;
-- 查看BE状态
show backends;
-- 查看Broker
show broker;
-- 查看集群均衡总进度
SHOW PROC "/cluster_balance";
-- 查看FE配置
SHOW FRONTEND CONFIG;
-- 查看BE配置
SHOW BACKEND CONFIG;
-- 查看磁盘冷热分层信息
SHOW PROC "/backends";
-- 查看正在运行的查询
SHOW PROCESSLIST;
-- 查看 BE 识别到的存储介质信息(SSD/HDD)
SHOW BACKEND CONFIG LIKE 'storage_root_path';




# 5.5 三台doris用户启动Doris-Broker,node1注册broker
#三台分别执行
su - doris
cd /opt/software/apache-doris-4.0.8-bin-x64/broker
./bin/start_broker.sh --daemon

# node1执行注册
# 建议用sql方式
mysql -h172.18.206.64 -P9030 -uroot -pDorisPassw0rd
ALTER SYSTEM ADD BROKER broker_hdfs "172.18.206.64:8000","172.18.206.65:8000","172.18.206.66:8000";

curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_broker?host=172.18.206.64&port=8000"
curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_broker?host=172.18.206.65&port=8000"
curl -u root:DorisPassw0rd -XPOST "http://172.18.206.64:8030/api/bootstrap/add_broker?host=172.18.206.66&port=8000"





# 步骤 6 扩容Kafka三节点KRaft
#node1 kafka用户停止kafka
su - kafka
/opt/software/kafka_2.13-3.7.0/bin/kafka-server-stop.sh

#三台root清空kafka数据目录
rm -rf /data/hdd/kafka/*
chown kafka:kafka /data/hdd/kafka
ls /data/hdd/kafka
# 输出为空,代表清理成功

# `controller.quorum.voters` 里面的数字**一定是node.id,不是broker.id**,写错集群无法启动。
# KRaft 支持两种部署形态:
# 1. **混合部署**:所有节点`broker+controller`,适合3节点小规模集群。
# 2. **分离部署**:3个controller节点(只做元数据,不存业务消息)+若干broker节点(只存消息,不参与选举)
#    -controller节点:只配置node.id,**不需要broker.id**
#    -broker节点:只配置broker.id,**不需要node.id**
# Kafka 3.7 版本限制:当 `process.roles=broker,controller`(混合角色),要求`node.id==broker.id`！

# node1 /opt/software/kafka_2.13-3.7.0/config/server.properties
process.roles=broker,controller
node.id=1
broker.id=1
controller.quorum.voters=1@172.18.206.64:9093,2@172.18.206.65:9093,3@172.18.206.66:9093
listeners=PLAINTEXT://:9092,CONTROLLER://:9093
inter.broker.listener.name=PLAINTEXT
advertised.listeners=PLAINTEXT://172.18.206.64:9092
controller.listener.names=CONTROLLER
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
log.dirs=/data/hdd/kafka
# 禁止客户端自动创建topic
auto.create.topics.enable=false
# 消息至少同步到2个副本才ack(3副本场景)不写,用默认1
# min.insync.replicas=2
# 消息保留时间,按需调整,这里30天不配置log.retention.hours,Kafka 3.7默认保留7天(168小时)
# log.retention.hours=720
# 分区副本leader均衡
auto.leader.rebalance.enable=true



# node2 /opt/software/kafka_2.13-3.7.0/config/server.properties
process.roles=broker,controller
node.id=2
broker.id=2
controller.quorum.voters=1@172.18.206.64:9093,2@172.18.206.65:9093,3@172.18.206.66:9093
listeners=PLAINTEXT://:9092,CONTROLLER://:9093
inter.broker.listener.name=PLAINTEXT
advertised.listeners=PLAINTEXT://172.18.206.65:9092
controller.listener.names=CONTROLLER
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
log.dirs=/data/hdd/kafka
# 禁止客户端自动创建topic
auto.create.topics.enable=false
# 消息至少同步到2个副本才ack(3副本场景)不写,用默认1
# min.insync.replicas=2
# 消息保留时间,按需调整,这里30天 不配置log.retention.hours,Kafka 3.7默认保留7天(168小时)
# log.retention.hours=720
# 分区副本leader均衡
auto.leader.rebalance.enable=true


# node3 /opt/software/kafka_2.13-3.7.0/config/server.properties
process.roles=broker,controller
node.id=3
broker.id=3
controller.quorum.voters=1@172.18.206.64:9093,2@172.18.206.65:9093,3@172.18.206.66:9093
listeners=PLAINTEXT://:9092,CONTROLLER://:9093
inter.broker.listener.name=PLAINTEXT
advertised.listeners=PLAINTEXT://172.18.206.66:9092
controller.listener.names=CONTROLLER
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
log.dirs=/data/hdd/kafka
# 禁止客户端自动创建topic
auto.create.topics.enable=false
# 消息至少同步到2个副本才ack(3副本场景)不写,用默认1
# min.insync.replicas=2
# 消息保留时间,按需调整,这里30天 不配置 log.retention.hours,Kafka 3.7默认保留7天(168小时)
# log.retention.hours=720
# 分区副本leader均衡
auto.leader.rebalance.enable=true


# 仅node1 kafka用户执行格式化
su - kafka
cd /opt/software/kafka_2.13-3.7.0
# 先执行这一句,拿到uuid,复制保存！ uU0errJWQtOjlmxMWO0RpA
bin/kafka-storage.sh random-uuid
# bin/kafka-storage.sh format -t $(bin/kafka-storage.sh random-uuid) -c config/server.properties
# 然后执行format,把上面输出的uuid替换进去
bin/kafka-storage.sh format -t "uU0errJWQtOjlmxMWO0RpA" -c config/server.properties

# node2 执行(**不要random-uuid,复用同一个uuid**)
su - kafka
cd /opt/software/kafka_2.13-3.7.0
bin/kafka-storage.sh format -t "uU0errJWQtOjlmxMWO0RpA" -c config/server.properties

# node3 执行(同样复用uuid)
su - kafka
cd /opt/software/kafka_2.13-3.7.0
bin/kafka-storage.sh format -t "uU0errJWQtOjlmxMWO0RpA" -c config/server.properties

# 三台kafka用户依次启动 kafka
su - kafka
cd /opt/software/kafka_2.13-3.7.0
# node1
bin/kafka-server-start.sh -daemon config/server.properties
# 等待一会,看日志选举成功,再启动node2,再node3
bin/kafka-server-start.sh -daemon config/server.properties
# node3
bin/kafka-server-start.sh -daemon config/server.properties

# 查看KRaft quorum leader
bin/kafka-metadata-quorum.sh --bootstrap-server 172.18.206.64:9092 describe --status
# ClusterId:              uU0errJWQtOjlmxMWO0RpA
# LeaderId:               1        # node1 当选Controller主节点
# CurrentVoters:          [1,2,3]  # 3个投票节点全部在线
# MaxFollowerLag:         0        # 从节点无延迟,元数据同步正常

# 查看broker列表
bin/kafka-broker-api-versions.sh --bootstrap-server 172.18.206.64:9092
# 查看消费组
bin/kafka-consumer-groups.sh --bootstrap-server 172.18.206.64:9092 --list
# 查看消费位点
bin/kafka-consumer-groups.sh --bootstrap-server 172.18.206.64:9092 --describe --group test_group
# 删除topic
bin/kafka-topics.sh --delete --topic test_msg --bootstrap-server 172.18.206.64:9092

#创建测试Topic(3分区,3副本,min.insync.replicas=2)
bin/kafka-topics.sh --create --topic test_msg \
--bootstrap-server 172.18.206.64:9092,172.18.206.65:9092,172.18.206.66:9092 \
--partitions 3 --replication-factor 3 \
--config min.insync.replicas=2
# 查看topic详情:
bin/kafka-topics.sh --describe --topic test_msg --bootstrap-server 172.18.206.64:9092
# 生产者(开一个窗口发消息)输入内容:{"id":1,"name":"test1"}
bin/kafka-console-producer.sh --topic test_msg --bootstrap-server 172.18.206.64:9092
# 消费者(新开窗口消费)
bin/kafka-console-consumer.sh --topic test_msg --bootstrap-server 172.18.206.64:9092 --from-beginning



注意点:
1. doris全部进程使用操作系统doris用户;kafka全部进程使用操作系统kafka用户
2. FE集群使用add_follower,**禁止使用add_observer**
3. Doris-Broker独立进程,用于hdfs/hive外表;RoutineLoad消费kafka不需要Doris-Broker
4. curl访问集群需要带上密码`-u root:DorisPassw0rd`



# FE/BE系统内核、limits配置
# 三台机器都要配置 `/etc/security/limits.conf`  vi /etc/security/limits.conf
doris soft nofile 655350
doris hard nofile 655350
doris soft nproc 655350
doris hard nproc 655350
kafka soft nofile 655350
kafka hard nofile 655350
kafka soft nproc 655350
kafka hard nproc 655350

# 如果自带值大于配置值则不管
# End of file
* soft nofile 1048576
* hard nofile 1048576
* soft nproc 1048576
* hard nproc 1048576

# 校验
# 切换到doris用户执行
su - doris
ulimit -n
ulimit -u


#内核参数`/etc/sysctl.conf`
vm.max_map_count=2000000
net.core.somaxconn=65535
net.ipv4.tcp_syncookies=1

# 生效:
sysctl -p
# 验证
sysctl vm.max_map_count
sysctl net.core.somaxconn
# 不配置高并发场景会出现文件句柄耗尽、OOM、BE异常crash。

# 快速校验脚本
# 校验内核参数
sysctl vm.max_map_count net.core.somaxconn net.ipv4.tcp_syncookies
# 校验doris用户ulimit
su - doris -c "ulimit -n; ulimit -u"
# 校验kafka用户ulimit
su - kafka -c "ulimit -n; ulimit -u"


# 集群启停脚本
#doris stop(doris用户)
/opt/software/apache-doris-4.0.8-bin-x64/fe/bin/stop_fe.sh
/opt/software/apache-doris-4.0.8-bin-x64/be/bin/stop_be.sh
/opt/software/apache-doris-4.0.8-bin-x64/broker/bin/stop_broker.sh

#kafka stop(kafka用户)
# 启动
/opt/software/kafka_2.13-3.7.0/bin/kafka-server-start.sh -daemon /opt/software/kafka_2.13-3.7.0/config/kraft/server.properties
# 停止
/opt/software/kafka_2.13-3.7.0/bin/kafka-server-stop.sh


# 启动顺序
# 先启动FE(元数据服务,必须先起来)
cd /opt/software/apache-doris-4.0.8-bin-x64/fe
./bin/start_fe.sh --daemon
# 等待FE完全就绪(等30‑60s,mysql可以连上9030)
# 再启动所有BE(存储计算节点)
cd /opt/software/apache-doris-4.0.8-bin-x64/be
numactl --interleave=all ./bin/start_be.sh --daemon
# BE启动后,在FE执行`ALTER SYSTEM ADD BACKEND`注册BE,BE才会和FE建立心跳。
# 停止顺序
# 先停止所有BE
cd /opt/software/apache-doris-4.0.8-bin-x64/be
./bin/stop_be.sh
# 再停止FE
cd /opt/software/apache-doris-4.0.8-bin-x64/fe
./bin/stop_fe.sh

# 查看FE进程
ps -ef | grep doris_fe
# 查看BE进程
ps -ef | grep doris_be
# FE侧看集群状态
mysql -h127.0.0.1 -P9030 -uroot -pDorisPassw0rd -e "SHOW BACKENDS;"
# 检查 kafka 进程
ps aux | grep kafka