ubuntu24.04部署单节点kafka_2.13-3.8.1

ubuntu24.04部署单节点kafka_2.13-3.8.1

下载地址推荐使用清华镜像源下载

https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/3.8.1/kafka_2.13-3.8.1.tgz

部署kafka部署

复制代码
# 解压kafka压缩包
sudo tar -zxvf kafka_2.13-3.8.1.tgz -C /usr/local/

# 改变权限及所有权
sudo chown -R hadoop:hadoopeco kafka_2.13-3.8.1/

# 启动zookeeper
/usr/local/kafka_2.13-3.8.1$ ./bin/zookeeper-server-start.sh ./config/zookeeper.properties

# 启动kafka
./bin/kafka-server-start.sh ./config/server.properties &

# kafka操作
./bin/kafka-topics.sh --bootstrap-server lh007:9092 --list

./bin/kafka-topics.sh --bootstrap-server lh007:9092 --create --replication-factor 1 --partitions 1 --topic test

./bin/kafka-console-producer.sh --bootstrap-server lh007:9092 --topic test

./bin/kafka-console-consumer.sh --bootstrap-server lh007:9092 --topic test

设置一键启动脚本

复制代码
#!/bin/bash

# 定义Kafka安装目录
KAFKA_HOME="/usr/local/kafka_2.13-3.8.1"
PID_FILE="kafka_pids.txt"

# 函数:检查命令执行是否成功
check_command() {
    if [ $? -ne 0 ]; then
        echo "Error: Failed to execute the last command."
        exit 1
    fi
}

# 函数:启动Zookeeper,并等待其准备就绪
start_zookeeper() {
    echo "Starting Zookeeper..."
    $KAFKA_HOME/bin/zookeeper-server-start.sh $KAFKA_HOME/config/zookeeper.properties &
    ZOOKEEPER_PID=$!

    # 等待几秒钟以确保Zookeeper已经启动
    sleep 5

    # 检查Zookeeper进程是否存在
    if ! ps -p $ZOOKEEPER_PID > /dev/null; then
        echo "Error: Zookeeper failed to start."
        exit 1
    else
        echo "Zookeeper started with PID: $ZOOKEEPER_PID"
    fi
}

# 函数:启动Kafka,并等待其准备就绪
start_kafka() {
    echo "Starting Kafka..."
    $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties &
    KAFKA_PID=$!

    # 等待几秒钟以确保Kafka已经启动
    sleep 5

    # 检查Kafka进程是否存在
    if ! ps -p $KAFKA_PID > /dev/null; then
        echo "Error: Kafka broker failed to start."
        exit 1
    else
        echo "Kafka broker has been started with PID: $KAFKA_PID"
    fi
}

# 函数:停止Zookeeper和Kafka
stop_services() {
    if [ -f "$PID_FILE" ]; then
        while IFS= read -r line; do
            kill -9 "$line" 2>/dev/null && echo "Stopped service with PID: $line"
        done < "$PID_FILE"
        rm -f "$PID_FILE"
        echo "All services have been stopped."
    else
        echo "No PID file found, services may not be running."
    fi
}

# 主逻辑
if [ $# -eq 0 ]; then
    ACTION="start"
else
    ACTION="$1"
fi

case "$ACTION" in
    start)
        start_zookeeper
        start_kafka

        # 保存PIDs到文件,以便以后可以停止服务
        echo $ZOOKEEPER_PID > "$PID_FILE"
        echo $KAFKA_PID >> "$PID_FILE"

        echo "PIDs have been saved to $PID_FILE for stopping services later."

        check_command
        echo "All services have been started successfully."
        ;;
    stop)
        stop_services
        ;;
    *)
        echo "Usage: $0 {start|stop}"
        exit 1
        ;;
esac

操作日志

复制代码
hadoop@lh007:~/scripts$ ./kafka_control.sh start
Starting Zookeeper...
[2024-12-11 09:23:54,782] INFO Reading configuration from: /usr/local/kafka_2.13-3.8.1/config/zookeeper.properties (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,788] INFO clientPortAddress is 0.0.0.0:2181 (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,788] INFO secureClientPort is not set (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,789] INFO observerMasterPort is not set (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,789] INFO metricsProvider.className is org.apache.zookeeper.metrics.impl.DefaultMetricsProvider (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,792] INFO autopurge.snapRetainCount set to 3 (org.apache.zookeeper.server.DatadirCleanupManager)
[2024-12-11 09:23:54,792] INFO autopurge.purgeInterval set to 0 (org.apache.zookeeper.server.DatadirCleanupManager)
[2024-12-11 09:23:54,792] INFO Purge task is not scheduled. (org.apache.zookeeper.server.DatadirCleanupManager)
[2024-12-11 09:23:54,792] WARN Either no config or no quorum defined in config, running in standalone mode (org.apache.zookeeper.server.quorum.QuorumPeerMain)
[2024-12-11 09:23:54,794] INFO Log4j 1.2 jmx support not found; jmx disabled. (org.apache.zookeeper.jmx.ManagedUtil)
[2024-12-11 09:23:54,795] INFO Reading configuration from: /usr/local/kafka_2.13-3.8.1/config/zookeeper.properties (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,796] INFO clientPortAddress is 0.0.0.0:2181 (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,796] INFO secureClientPort is not set (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,796] INFO observerMasterPort is not set (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,796] INFO metricsProvider.className is org.apache.zookeeper.metrics.impl.DefaultMetricsProvider (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
[2024-12-11 09:23:54,796] INFO Starting server (org.apache.zookeeper.server.ZooKeeperServerMain)
[2024-12-11 09:23:54,815] INFO ServerMetrics initialized with provider org.apache.zookeeper.metrics.impl.DefaultMetricsProvider@4590c9c3 (org.apache.zookeeper.server.ServerMetrics)
[2024-12-11 09:23:54,818] INFO ACL digest algorithm is: SHA1 (org.apache.zookeeper.server.auth.DigestAuthenticationProvider)
[2024-12-11 09:23:54,818] INFO zookeeper.DigestAuthenticationProvider.enabled = true (org.apache.zookeeper.server.auth.DigestAuthenticationProvider)
[2024-12-11 09:23:54,822] INFO zookeeper.snapshot.trust.empty : false (org.apache.zookeeper.server.persistence.FileTxnSnapLog)
[2024-12-11 09:23:54,835] INFO  (org.apache.zookeeper.server.ZooKeeperServer)
[2024-12-11 09:23:54,836] INFO   ______                  _                                           (org.apache.zookeeper.server.ZooKeeperServer)
[2024-12-11 09:23:54,836] INFO  |___  /                 | |                                          (org.apache.zookeeper.server.ZooKeeperServer)
[2024-12-11 09:23:54,836] INFO     / /    ___     ___   | | __   ___    ___   _ __     ___   _ __    (org.apache.zookeeper.server.ZooKeeperServer)
[2024-12-11 09:23:54,836] INFO    / /    / _ \   / _ \  | |/ /  / _ \  / _ \ | '_ \   / _ \ | '__| (org.apache.zookeeper.server.ZooKeeperServer)
[2024-12-11 09:23:54,836] INFO   / /__  | (_) | | (_) | |   <  |  __/ |  __/ | |_) | |  __/ | |     (org.apache.zookeeper.server.ZooKeeperServer)
[2024-12-11 09:23:54,836] INFO  /_____|  \___/   \___/  |_|\_\  \___|  \___| | .__/   \___| |_| (org.apache.zookeeper.server.ZooKeeperServer)

 epoch 0, of which 47 milliseconds was spent in the scheduler. (kafka.coordinator.group.GroupMetadataManager)
Kafka broker has been started with PID: 199769
PIDs have been saved to kafka_pids.txt for stopping services later.
All services have been started successfully.
hadoop@lh007:~/scripts$ jps
200310 Jps
199769 Kafka
199295 QuorumPeerMain
hadoop@lh007:~/scripts$ cat kafka_control.sh
#!/bin/bash

操作脚本

复制代码
hadoop@lh007:~/scripts$ cat my_kafka_options_en.sh
#!/bin/bash

# 定义Kafka安装目录和其他常量
KAFKA_HOME="/usr/local/kafka_2.13-3.8.1"
BOOTSTRAP_SERVER="lh007:9092"

# 函数:显示帮助信息
show_help() {
    echo "Usage: $0 {list|create|produce|consume} [options]"
    echo "Commands:"
    echo "  list       List all topics."
    echo "  create     Create a new topic with specified replication factor and partitions."
    echo "  produce    Start a console producer to send messages to the specified topic."
    echo "  consume    Start a console consumer to receive messages from the specified topic."
    echo "Options for 'create':"
    echo "  --replication-factor <value>"
    echo "  --partitions <value>"
    echo "  --topic <name>"
    echo "Options for 'produce' and 'consume':"
    echo "  --topic <name>"
}

# 主逻辑
case "$1" in
    list)
        shift
        $KAFKA_HOME/bin/kafka-topics.sh --bootstrap-server $BOOTSTRAP_SERVER --list
        ;;
    create)
        shift
        if [ $# -lt 6 ]; then
            echo "Error: Not enough arguments for creating a topic."
            show_help
            exit 1
        fi
        while [[ $# -gt 0 ]]; do
            case "$1" in
                --replication-factor)
                    REPLICATION_FACTOR="$2"
                    shift 2
                    ;;
                --partitions)
                    PARTITIONS="$2"
                    shift 2
                    ;;
                --topic)
                    TOPIC_NAME="$2"
                    shift 2
                    ;;
                *)
                    echo "Unknown option: $1"
                    show_help
                    exit 1
                    ;;
            esac
        done
        $KAFKA_HOME/bin/kafka-topics.sh --bootstrap-server $BOOTSTRAP_SERVER --create --replication-factor $REPLICATION_FACTOR --partitions $PARTITIONS --topic $TOPIC_NAME
        ;;
    produce)
        shift
        while [[ $# -gt 0 ]]; do
            case "$1" in
                --topic)
                    TOPIC_NAME="$2"
                    shift 2
                    ;;
                *)
                    echo "Unknown option: $1"
                    show_help
                    exit 1
                    ;;
            esac
        done
        if [ -z "$TOPIC_NAME" ]; then
            echo "Error: Topic name is required for producing messages."
            show_help
            exit 1
        fi
        $KAFKA_HOME/bin/kafka-console-producer.sh --bootstrap-server $BOOTSTRAP_SERVER --topic $TOPIC_NAME
        ;;
    consume)
        shift
        while [[ $# -gt 0 ]]; do
            case "$1" in
                --topic)
                    TOPIC_NAME="$2"
                    shift 2
                    ;;
                *)
                    echo "Unknown option: $1"
                    show_help
                    exit 1
                    ;;
            esac
        done
        if [ -z "$TOPIC_NAME" ]; then
            echo "Error: Topic name is required for consuming messages."
            show_help
            exit 1
        fi
        $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server $BOOTSTRAP_SERVER --topic $TOPIC_NAME --from-beginning
        ;;
    *)
        show_help
        exit 1
        ;;
esac

中文版脚本

复制代码
hadoop@lh007:~/scripts$ cat my_kafka_options_zn.sh
#!/bin/bash

# 定义Kafka安装目录和其他常量
KAFKA_HOME="/usr/local/kafka_2.13-3.8.1"
BOOTSTRAP_SERVER="lh007:9092"

# 函数:显示帮助信息(中文)
show_help() {
    echo "用法: $0 {list|create|produce|consume} [选项]"
    echo "命令:"
    echo "  list       列出所有主题."
    echo "  create     创建一个新主题,并指定复制因子和分区数."
    echo "  produce    启动控制台生产者,向指定主题发送消息."
    echo "  consume    启动控制台消费者,从指定主题接收消息."
    echo "创建主题的选项:"
    echo "  --replication-factor <值>   设置复制因子"
    echo "  --partitions <值>           设置分区数量"
    echo "  --topic <名称>              指定主题名称"
    echo "启动生产和消费的选项:"
    echo "  --topic <名称>              指定主题名称"
}

# 主逻辑
case "$1" in
    list)
        shift
        echo "正在列出所有主题..."
        $KAFKA_HOME/bin/kafka-topics.sh --bootstrap-server $BOOTSTRAP_SERVER --list
        ;;
    create)
        shift
        if [ $# -lt 6 ]; then
            echo "错误: 创建主题时参数不足."
            show_help
            exit 1
        fi
        while [[ $# -gt 0 ]]; do
            case "$1" in
                --replication-factor)
                    REPLICATION_FACTOR="$2"
                    shift 2
                    ;;
                --partitions)
                    PARTITIONS="$2"
                    shift 2
                    ;;
                --topic)
                    TOPIC_NAME="$2"
                    shift 2
                    ;;
                *)
                    echo "未知选项: $1"
                    show_help
                    exit 1
                    ;;
            esac
        done
        echo "正在创建新主题: $TOPIC_NAME, 复制因子: $REPLICATION_FACTOR, 分区数: $PARTITIONS..."
        $KAFKA_HOME/bin/kafka-topics.sh --bootstrap-server $BOOTSTRAP_SERVER --create --replication-factor $REPLICATION_FACTOR --partitions $PARTITIONS --topic $TOPIC_NAME
        ;;
    produce)
        shift
        while [[ $# -gt 0 ]]; do
            case "$1" in
                --topic)
                    TOPIC_NAME="$2"
                    shift 2
                    ;;
                *)
                    echo "未知选项: $1"
                    show_help
                    exit 1
                    ;;
            esac
        done
        if [ -z "$TOPIC_NAME" ]; then
            echo "错误: 发送消息时必须指定主题名称."
            show_help
            exit 1
        fi
        echo "正在启动控制台生产者,向主题 '$TOPIC_NAME' 发送消息..."
        $KAFKA_HOME/bin/kafka-console-producer.sh --bootstrap-server $BOOTSTRAP_SERVER --topic $TOPIC_NAME
        ;;
    consume)
        shift
        while [[ $# -gt 0 ]]; do
            case "$1" in
                --topic)
                    TOPIC_NAME="$2"
                    shift 2
                    ;;
                *)
                    echo "未知选项: $1"
                    show_help
                    exit 1
                    ;;
            esac
        done
        if [ -z "$TOPIC_NAME" ]; then
            echo "错误: 接收消息时必须指定主题名称."
            show_help
            exit 1
        fi
        echo "正在启动控制台消费者,从主题 '$TOPIC_NAME' 接收消息..."
        $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server $BOOTSTRAP_SERVER --topic $TOPIC_NAME --from-beginning
        ;;
    *)
        show_help
        exit 1
        ;;
esac

**欢迎关注、您的支持与回复,是我继续开发更多技术文档分享的动力。欢迎扫描关注WX:**夏目的杂货店

相关推荐
牛马小陈同学4 小时前
Kafka+Zookeeper从docker部署到spring boot使用完整教程
linux·spring boot·docker·zookeeper·kafka·prettyzoo·kafka-ui
IT成长日记4 小时前
【Kafka基础】ZooKeeper在Kafka中的核心作用:分布式系统中枢神经系统
zookeeper·kafka·zk用途
程序员 小柴4 小时前
RabbitMQ的工作模式
分布式·rabbitmq·ruby
蒋星熠4 小时前
在VMware下Hadoop分布式集群环境的配置--基于Yarn模式的一个Master节点、两个Slaver(Worker)节点的配置
大数据·linux·hadoop·分布式·ubuntu·docker
小样vvv5 小时前
【分布式】微服务系统中基于 Hystrix 的熔断实现方案
分布式·hystrix·微服务
清风19818 小时前
kafka消息可靠性传输语义
数据库·分布式·kafka
小诸葛的博客8 小时前
Kafka、RocketMQ、Pulsar对比
分布式·kafka·rocketmq
数据智能老司机10 小时前
CockroachDB权威指南——SQL调优
数据库·分布式·架构
数据智能老司机10 小时前
CockroachDB权威指南——应用设计与实现
数据库·分布式·架构