linux端进行kafka集群服务的搭建

1.环境变量的配置

1.1 修改/etc/hosts

js 复制代码

192.168.220.102 hadoop102
192.168.220.103 hadoop103
192.168.220.104 hadoop104

1.2 修改/etc/hostname

js 复制代码

hadoop102

1.3 添加文件/etc/profile.d/my_env.sh

js 复制代码

#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin
#HADOOP_HOME
export HADOOP_HOME=/opt/module/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
#KAFKA_HOME
export KAFKA_HOME=/opt/module/kafka
export PATH=$PATH:$KAFKA_HOME/bin

保存之后 source /etc/profile.d/my_env.sh

2.SSH之间进行无密登录访问

js 复制代码

1.使用hadoop102 进行演示

登录虚拟机hadoop102
ssh-keygen -t rsa 
然后敲（三个回车），就会生成两个文件id_rsa（私钥）、id_rsa.pub（公钥）

ssh-copy-id hadoop102
ssh-copy-id hadoop103
ssh-copy-id hadoop104

3.zookeeper安装

js 复制代码

在/opt/module/zookeeper/目录下创建zkData
# 创建myid文件内容如下
1

# 进入cd /opt/module/zookeeper/conf文件目录
cd /opt/module/zookeeper/conf

# 修改文件名称
mv zoo_sample.cfg zoo.cfg

# 修改文件内容
vim zoo.cfg

# 以下内容为修改内容
dataDir=/opt/module/zookeeper/zkData
server.1=hadoop102:2888:3888
server.2=hadoop103:2888:3888
server.3=hadoop104:2888:3888

# 进入/opt/module路径
cd /opt/module

# 调用分发脚本将本机得ZooKeeper安装包分发到其他两台机器
xsync zookeeper

4.zookeeper启动脚本

js 复制代码

#!/bin/bash

case $1 in
"start"){
	for i in hadoop102 hadoop103 hadoop104
	do
        echo ---------- zookeeper $i 启动 ------------
		ssh $i "/opt/module/zookeeper/bin/zkServer.sh start"
	done
};;
"stop"){
	for i in hadoop102 hadoop103 hadoop104
	do
        echo ---------- zookeeper $i 停止 ------------    
		ssh $i "/opt/module/zookeeper/bin/zkServer.sh stop"
	done
};;
"status"){
	for i in hadoop102 hadoop103 hadoop104
	do
        echo ---------- zookeeper $i 状态 ------------    
		ssh $i "/opt/module/zookeeper/bin/zkServer.sh status"
	done
};;
esac

5.kafka安装

js 复制代码

vim server.properties

#broker的全局唯一编号，每个服务节点不能重复，只能是数字。
broker.id=1 #(hadoop102->1 hadoop103->2 hadoop104->3)

#broker对外暴露的IP和端口 （每个节点单独配置）
advertised.listeners=PLAINTEXT://hadoop102:9092 #(hadoop102->hadoop102:9092 hadoop103->hadoop103:9092 hadoop104->hadoop104:9092)

路径，路径与路径之间可以用"，"分隔
log.dirs=/opt/module/kafka/datas

#配置连接Zookeeper集群地址（在zk根目录下创建/kafka，方便管理）
zookeeper.connect=hadoop102:2181,hadoop103:2181,hadoop104:2181/kafka

6.kafka启动脚本

js 复制代码

#! /bin/bash

case $1 in
"start"){
    for i in hadoop102 hadoop103 hadoop104
    do
        echo " --------启动 $i Kafka-------"
        ssh $i "/opt/module/kafka/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties"
    done
};;
"stop"){
    for i in hadoop102 hadoop103 hadoop104
    do
        echo " --------停止 $i Kafka-------"
        ssh $i "/opt/module/kafka/bin/kafka-server-stop.sh "
    done
};;
esac

7.cluster管理脚本

js 复制代码

#!/bin/bash

case $1 in
"start"){
        echo ================== 启动 Kafka集群 ==================
        #启动 Zookeeper集群
        zk.sh start

        #启动 Kafka采集集群
        kafka.sh start

        };;
"stop"){
        echo ================== 停止 Kafka集群 ==================

        #停止 Kafka采集集群
        kafka.sh stop

		#循环直至 Kafka 集群进程全部停止
		kafka_count=$(xcall jps | grep Kafka | wc -l)
		while [ $kafka_count -gt 0 ]
		do
			sleep 1
			kafka_count=$(xcall | grep Kafka | wc -l)
            echo "当前未停止的 Kafka 进程数为 $kafka_count"
		done

        #停止 Zookeeper集群
        zk.sh stop
};;
esac

8.xcall与xsync脚本

8.1 xcall

js 复制代码

#!/bin/bash

# ====== 节点列表（按你集群实际修改）======
HOSTS=("hadoop102" "hadoop103" "hadoop104")

# ====== 参数校验 ======
if [ $# -lt 1 ]; then
  echo "Usage: xcall <command>"
  exit 1
fi

# ====== 执行 ======
for host in "${HOSTS[@]}"; do
  echo "==================== $host ===================="
  ssh $host "$@"
done

8.2 xsync

js 复制代码

#!/bin/bash

# 1. 判断参数个数
if [ $# -lt 1 ]; then
    echo "Not Enough Arguement!"
    exit 1
fi

# 2. 遍历集群所有机器
for host in hadoop102 hadoop103 hadoop104; do
    echo "==================== $host ===================="

    # 3. 遍历所有目录，挨个发送
    for file in "$@"; do
        # 4. 判断文件是否存在
        if [ -e "$file" ]; then
            # 5. 获取父目录
            pdir=$(cd -P "$(dirname "$file")"; pwd)
            # 6. 获取当前文件的名称
            fname=$(basename "$file")

            ssh "$host" "mkdir -p $pdir"
            rsync -av "$pdir/$fname" "$host:$pdir"
        else
            echo "$file does not exist!"
        fi
    done
done