Kubeadm 安装 Kubernetes 高可用集群 v1.30.0

1、修改主机名（各个节点）

bash 复制代码

hostnamectl set-hostname xxx

2、hosts 文件加入主机名（全部节点）

cat /etc/hosts

bash 复制代码

192.168.88.5 master1
192.168.88.6 master2
192.168.88.7 master3
192.168.88.8 node1

3、关闭防火墙（全部节点）

bash 复制代码

systemctl stop firewalld && systemctl disable firewalld

4、关闭 selinux（全部节点，需要重启服务器）

bash 复制代码

sed -i 's/^ *SELINUX=enforcing/SELINUX=disabled/g'

5、关闭各节点 swap 分区（全部节点，需要重启服务器）

bash 复制代码

sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

6、同步时间（全部节点）

bash 复制代码

yum -y install chrony && systemctl enable chronyd && systemctl start chronyd

bash 复制代码

timedatectl set-timezone Asia/Shanghai && chronyc -a makestep

7、安装 ipset 服务（全部节点）

bash 复制代码

yum -y install ipvsadm ipset sysstat conntrack libseccomp

8、内核调整,将桥接的 IPv4 流量传递到 iptables 的链（全部节点）

cat /etc/sysctl.d/k8s.conf

bash 复制代码

net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1

文件立即生效

bash 复制代码

sysctl -p /etc/sysctl.d/k8s.conf

9、开启 ipvs 模块（全部节点）

cat /etc/sysconfig/modules/ipvs.modules

bash 复制代码

#!/bin/sh
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4

10、赋予权限并执行脚本，用过滤查看是否生效（全部节点）

bash 复制代码

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

11、安装 haproxy 和 keepalived（所有节点）

bash 复制代码

yum -y install keepalived haproxy

12、修改 master 节点 keepalived.conf（全部节点）

cat /etc/keepalived/keepalived.conf（master1 节点）

bash 复制代码

! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
    script_user root
    enable_script_security
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"   # 脚本检测路径
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance VI_1 {
    state  MASTER           # 第一台 master 节点是 MASTER，其他 meater 节点是 BACKUP
    interface eno16777736   # 本机网卡名
    virtual_router_id 51    # 路由都是一样
    priority 100            # 第一台权重是 100
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.88.3        # VIP IP
    }
    track_script {
        check_apiserver     # 虚拟检测模块
    }
}

cat /etc/keepalived/keepalived.conf （master2 节点）

bash 复制代码

! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
    script_user root
    enable_script_security
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"   # 脚本检测路径
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance VI_1 {
    state  BACKUP           # 第一台 master 节点是 MASTER，其他 meater 节点是 BACKUP
    interface eno16777736   # 本机网卡名
    virtual_router_id 51    # 路由都是一样
    priority 90            # 第一台权重是 100
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.88.3        # VIP IP
    }
    track_script {
        check_apiserver     # 虚拟检测模块
    }
}

cat /etc/keepalived/keepalived.conf（master3 节点）

bash 复制代码

! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
    script_user root
    enable_script_security
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"   # 脚本检测路径
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance VI_1 {
    state  BACKUP           # 第一台 master 节点是 MASTER，其他 meater 节点是 BACKUP
    interface eno16777736   # 本机网卡名
    virtual_router_id 51    # 路由都是一样
    priority 80            # 第一台权重是 100
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.88.3        # VIP IP
    }
    track_script {
        check_apiserver     # 虚拟检测模块
    }
}

13、修改 haproxy.cfg 文件（三个 master 节点配置文件一样）

cat /etc/haproxy/haproxy.cfg

bash 复制代码

#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------
 
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2
 
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
 
    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats
 
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
 
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend  kubernetes-apiserver
    mode                        tcp
    bind                        *:8443
    option                      tcplog
    default_backend             kubernetes-apiserver
 
#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
listen stats
    bind            *:1080
    stats auth      admin:awesomePassword
    stats refresh   5s
    stats realm     HAProxy\ Statistics
    stats uri       /admin?stats
 
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
    mode        tcp
    balance     roundrobin
    server  master1 192.168.88.5:6443 check
    server  master2 192.168.88.6:6443 check
    server  master3 192.168.88.7:6443 check

14、VIP 检测脚本

cat /etc/keepalived/check_apiserver.sh

bash 复制代码

#!/bin/sh
# HAPROXY down
A=`ps -C haproxy --no-header | wc -l`
if [ $A -eq 0 ]
then
systmectl start haproxy
  if [ ps -C haproxy --no-header | wc -l -eq 0 ]
  then
    killall -9 haproxy
    echo "HAPROXY down" | mail -s "haproxy"
    sleep 3600
fi  
fi

给脚本添加执行权限

bash 复制代码

chmod +x /etc/keepalived/check_apiserver.sh

15、启动服务（启动之后自行查看服务是否报错）

bash 复制代码

systemctl enable keepalived && systemctl start keepalived

bash 复制代码

systemctl enable haproxy && systemctl start haproxy

16、下载 docker 源（全部节点）

bash 复制代码

wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

17、安装容器运行时（全部节点）

bash 复制代码

yum -y install docker-ce && systemctl enable docker && systemctl start docker

18、创建 daemon.json 配置文件（全部节点）

cat /etc/docker/daemon.json

bash 复制代码

{
    "exec-opts":["native.cgroupdriver=systemd"],
    "log-driver":"json-file",
    "log-opts":{
        "max-size":"100m"
    }
}

重启 docker

bash 复制代码

systemctl restart docker

19、安装cri-docker

bash 复制代码

curl -LO https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.13/cri-dockerd-0.3.13-3.el7.x86_64.rpm && rpm -ivh cri-dockerd-0.3.13-3.el7.x86_64.rpm

查看版本

bash 复制代码

cri-dockerd --version

加入开机启动并启动

bash 复制代码

systemctl enable cri-docker && systemctl start cri-docker

20、配置 k8s 源

cat /etc/yum.repo.d/k8s.repo

bash 复制代码

[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
enabled=1
gpgcheck=0

更新yum缓存

bash 复制代码

yum clean all && yum makecache

21、安装 kubeadm,kubelet,kubectl

bash 复制代码

yum -y install kubeadm-1.30.0 kubelet-1.30.0 kubectl-1.30.0 && systemctl enable kubelet

22、整合 kubelet 和 cri-docker

修改/usr/lib/systemd/system/cri-docker.service文件的ExecStart字段（全部节点）

bash 复制代码

--network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d

重启cri-dockerd服务

bash 复制代码

systemctl daemon-reload && systemctl restart cri-docker

23、生成初始化配置文件

kubeadm config print init-defaults > kubeadm-init.yaml

24、修改默认配置文件

cat kubeadm-init.yaml

bash 复制代码

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.88.5         # 当前主机 ip
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  name: master1      # 本机主机名                    
  taints:
  - effect: "NoSchedule"
    key: "node-role.kubernetes.io/master"
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.88.3:8443  # 重点！！ 这里是我们自定义的域名，端口是haproxy的端口，单master可注释掉这个参数
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.30.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16       # k8s 网卡网段（flannel，如果是 calico 需要改成别的网段）
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs     # k8s 需要 ipvs 模式
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd   # k8s 驱动 systemd

25、下载镜像

bash 复制代码

kubeadm config images pull --config kubeadm-init.yaml

26、初始化集群

bash 复制代码

kubeadm init --config kubeadm-init.yaml

27、在其他 master 节点创建目录

bash 复制代码

mkdir -p /etc/kubernetes/pki/etcd/

28、把主 master 节点证书分别复制到从 master 节点

cat cert.sh(自己创建脚本)

bash 复制代码

#!/bin/sh/
ip="192.168.88.6 192.168.88.7"

for host in $ip
do 
   scp /etc/kubernetes/pki/ca.* root@192.168.88.6:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/sa.* root@192.168.88.6:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/front-proxy-ca.* root@192.168.88.6:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/etcd/ca.* root@192.168.88.6:/etc/kubernetes/pki/etcd/
   scp /etc/kubernetes/admin.conf root@192.168.88.6:/etc/kubernetes/
   scp /etc/kubernetes/pki/ca.* root@192.168.88.7:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/sa.* root@192.168.88.7:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/front-proxy-ca.* root@192.168.88.7:/etc/kubernetes/pki/
   scp /etc/kubernetes/pki/etcd/ca.* root@192.168.88.7:/etc/kubernetes/pki/etcd/
   scp /etc/kubernetes/admin.conf root@192.168.88.7:/etc/kubernetes/
done

把主节点admin.conf证书复制到其他node节点

bash 复制代码

scp /etc/kubernetes/admin.conf root@192.168.88.8:/etc/kubernetes/

29、其他 master 节点加入集群命令

bash 复制代码

kubeadm join 192.168.88.3:6443 --token uozrup.6v02tswc3xk33org \
	--discovery-token-ca-cert-hash sha256:b5391b689f7986a2ef8aa84665568e9ea25a031c89c31472bb56f14e17c7c471 \
	--control-plane --certificate-key 3fdfbfe831ebc90939cdbde89d3894cbe527b64f67a4cd90de56018dfae75399 
    --cri-socket=unix:///var/run/cri-dockerd.sock

node 节点加入集群命令

bash 复制代码

kubeadm join 192.168.88.3:6443 --token uozrup.6v02tswc3xk33org \
	--discovery-token-ca-cert-hash sha256:b5391b689f7986a2ef8aa84665568e9ea25a031c89c31472bb56f14e17c7c471 --cri-socket=unix:///var/run/cri-dockerd.sock

30、非 root 用户加入 master 执行以下命令

bash 复制代码

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

root 用户加入 master 执行以下命令

bash 复制代码

export KUBECONFIG=/etc/kubernetes/admin.conf

31、安装网络插件

bash 复制代码

kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

查看集群网络是否正常

bash 复制代码

kubectl get pods --all-namespaces

bash 复制代码

kubectl get pods --all-namespaces

32、安装 etcdctl 客户端（可以用脚本中的集群查询命令也可以单独用最下面的 etcd 查询命令）

cat etcdctl.sh

bash 复制代码

#!/bin/sh
# Author: Jacket_San
# Email: xxx@163.com
# Date: 26/11/2024
# Filename: etcd.sh
 
##########################
#    etcd 客户端安装     #    
# etcdctl 查询数据库信息 #
##########################


# etcd 路径
path="/root/etcd/"


# yum 安装 jq
yum -y install jq

# 获取最新版本号
LATEST_VERSION=$(curl -s https://api.github.com/repos/etcd-io/etcd/releases/latest | jq -r .tag_name)

# 下载最新版本的 tar.gz 文件
curl -L -O https://github.com/etcd-io/etcd/releases/download/${LATEST_VERSION}/etc-${LATEST_VERSION}-linux-amd64.tar.gz 

# 下载 etcd 客户端并加入环境变量
if [ ! -d "$path" ];then
    mkdir $path
fi

 
mv etcd* /root/etcd/ && cd $path && tar -zxf *.tar.gz
mv */etcdctl /usr/local/bin && chmod +x /usr/local/bin/
 
# 查看 etcdctl 版本
etcdctl version && etcdctl -h

# 查看etcd高可用集群健康状态
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint health

# 查看etcd高可用集群列表
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 member list

# 查看etcd高可用集群leader
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint status

33、查看etcd高可用集群健康状态

bash 复制代码

ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint health

查看etcd高可用集群列表

bash 复制代码

ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 member list

查看etcd高可用集群leader

bash 复制代码

ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/peer.crt --key=/etc/kubernetes/pki/etcd/peer.key --write-out=table --endpoints=192.168.88.5:2379,192.168.88.6:2379,192.168.88.7:2379 endpoint status