二进制安装部署单master节点K8S集群v1.23.6版本

一、系统基本的配置

1.3 操作系统初始化配置

# 根据规划设置主机名
hostnamectl set-hostname k8s-master1 && bash  #master1上执行
hostnamectl set-hostname k8s-master2 && bash  #master2上执行
hostnamectl set-hostname k8s-node1   && bash  #node1上执行

# 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

# 关闭selinux
sed -i 's/enforcing/disabled/' /etc/selinux/config  # 永久
setenforce 0  # 临时

# 关闭swap
swapoff -a  # 临时
sed -ri 's/.*swap.*/#&/' /etc/fstab    # 永久

# 在master添加hosts
cat >> /etc/hosts << EOF
192.168.186.128 k8s-master1
192.168.186.129 k8s-master2
192.168.186.130 k8s-node1
EOF

# 将桥接的IPv4流量传递到iptables的链
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system  # 生效

# 时间同步
yum install ntpdate -y
ntpdate time.windows.com

二、部署Nginx+Keepalived高可用负载均衡器

2.1 安装软件包（主/备）

 yum install epel-release -y
 yum install nginx keepalived nginx-mod-stream -y
 

2.2 Nginx配置文件（主/备一样）

cp -r /etc/nginx/nginx.conf   /etc/nginx/nginx.conf.bak

cp -r /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

cat > /etc/nginx/nginx.conf << "EOF"
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

# 四层负载均衡，为两台Master apiserver组件提供负载均衡
stream {

    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';

    access_log  /var/log/nginx/k8s-access.log  main;

    upstream k8s-apiserver {
       server 192.168.186.128:6443;   # Master1 APISERVER IP:PORT
       server 192.168.186.129:6443;   # Master2 APISERVER IP:PORT
    }
    
    server {
       listen 16443;  # 由于nginx与master节点复用，这个监听端口不能是6443，否则会冲突
       proxy_pass k8s-apiserver;
    }
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    server {
        listen       80 default_server;
        server_name  _;

        location / {
        }
    }
}
EOF

2.3 keepalived配置文件（Nginx Master）

cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     [email protected] 
     [email protected] 
     [email protected] 
   } 
   notification_email_from [email protected]  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER
} 

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state MASTER 
    interface ens33  **# 修改为实际网卡名**
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 100    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    # 虚拟IP
    virtual_ipaddress { 
        192.168.186.188/24  #注意这个需要修改
    } 
    track_script {
        check_nginx
    } 
}
EOF


cat > /etc/keepalived/check_nginx.sh  << "EOF"
#!/bin/bash
count=$(ss -antp |grep 16443 |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi
EOF

chmod +x /etc/keepalived/check_nginx.sh

•vrrp_script：指定检查nginx工作状态脚本（根据nginx状态判断是否故障转移）

•virtual_ipaddress：虚拟IP（VIP）

准备上述配置文件中检查nginx运行状态的脚本

2.4 keepalived配置文件（Nginx Backup）

cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     [email protected] 
     [email protected] 
     [email protected] 
   } 
   notification_email_from [email protected]  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_BACKUP
} 

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state BACKUP 
    interface ens33
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 90
    advert_int 1
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.186.188/24
    } 
    track_script {
        check_nginx
    } 
}
EOF


cat > /etc/keepalived/check_nginx.sh  << "EOF"
#!/bin/bash
count=$(ss -antp |grep 16443 |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi
EOF

chmod +x /etc/keepalived/check_nginx.sh

注：keepalived根据脚本返回状态码（0为工作正常，非0不正常）判断是否故障转移。

2.5 启动并设置开机启动

#现在master1上执行
systemctl daemon-reload
systemctl start nginx
systemctl start keepalived
systemctl enable nginx
systemctl enable keepalived

2.6 查看keepalived工作状态

[root@k8s-master1 ~]# ip add  #看到VIP在master1上
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:9c:ac:21 brd ff:ff:ff:ff:ff:ff
    inet 192.168.186.128/24 brd 192.168.186.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet  192.168.186.188/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::97b9:35f:e366:62f8/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@k8s-master1 ~]# 
########可以看到，在ens33网卡绑定了192.168.186.188 虚拟IP，说明工作正常。

2.7 Nginx+Keepalived高可用测试

关闭主节点Nginx，测试VIP是否漂移到备节点服务器。 在Nginx Master执行 pkill nginx 在Nginx Backup，ip addr命令查看已成功绑定VIP。