1:拉去镜像osixia/keepalived
xml
[root@localhost ~]# docker pull osixia/keepalived
Using default tag: latest
latest: Pulling from osixia/keepalived
89d9c30c1d48: Pull complete
c9ada908efbe: Pull complete
4a9101e4ada9: Pull complete
b428f77d2dc4: Pull complete
c1f2db3054e3: Pull complete
8ef2fa7e2034: Pull complete
c641dd822a23: Pull complete
Digest: sha256:74179ae03efe2a975bfbd411181315d6eb8052d91c10627b03ad60bceac71890
Status: Downloaded newer image for osixia/keepalived:latest
docker.io/osixia/keepalived:latest
[root@localhost ~]#
2: 启动容器keepalived做为MASTER
xml
[root@localhost ~]# docker run -d --name keepalived-master --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -e KEEPALIVED_INTERFACE="ens33" -e KEEPALIVED_PRIORITY=100 -e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['192.168.11.43', '192.168.11.47']" -e KEEPALIVED_VIRTUAL_IPS="192.168.11.40" -e KEEPALIVED_STATE="MASTER" --privileged=true osixia/keepalived --loglevel debug
5bd11fe03cdcd2ebf35722066b030fd55b30b46fe23e6ae8caca5a3952f2ea72
[root@localhost ~]#
启动容器keepalived做为BACKUP
xml
[root@localhost opt]# docker run -d --name keepalived-backup --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -e KEEPALIVED_INTERFACE="ens33" -e KEEPALIVED_PRIORITY=50 -e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['192.168.11.43', '192.168.11.47']" -e KEEPALIVED_VIRTUAL_IPS="192.168.11.40" -e KEEPALIVED_STATE="BACKUP" --privileged=true osixia/keepalived --loglevel debug
e3de9120daaffea32c1ffc72424622b47bababef80e9da6dd0a99f8459788cfc
[root@localhost opt]#
如果MASTER和BACKUP的KEEPALIVED_PRIORITY一样,需要手动才能切换
2-1:--net=host这是共享宿主机网络,KEEPALIVED_INTERFACE="ens33"中的ens33需要确定宿主机是否有这个网卡,通过命令ip add查看
bash
[root@localhost ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:46:90:db brd ff:ff:ff:ff:ff:ff
inet 192.168.11.43/24 brd 192.168.11.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.11.40/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fd0d:3fd3:f1d::907/128 scope global noprefixroute
valid_lft forever preferred_lft forever
inet6 fd0d:3fd3:f1d:0:b544:377c:c1d2:978f/64 scope global noprefixroute
valid_lft forever preferred_lft forever
inet6 fe80::64d6:f397:92fd:1c0f/64 scope link noprefixroute
valid_lft forever preferred_lft forever
2-2:查看容器情况
bash
[root@localhost ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
5bd11fe03cdc osixia/keepalived "/container/tool/run..." 12 minutes ago Up 11 minutes keepalived-master
73e19612fe5f mysql:8.0.33 "docker-entrypoint.s..." 31 hours ago Up 4 hours 0.0.0.0:3306->3306/tcp, :::3306->3306/tcp, 33060/tcp mysql
2-3:通过IP-add查看是否有这个KEEPALIVED_VIRTUAL_IPS="192.168.11.40",有就代表正常
bash
[root@localhost ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:46:90:db brd ff:ff:ff:ff:ff:ff
inet 192.168.11.43/24 brd 192.168.11.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.11.40/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fd0d:3fd3:f1d::907/128 scope global noprefixroute
valid_lft forever preferred_lft forever
inet6 fd0d:3fd3:f1d:0:b544:377c:c1d2:978f/64 scope global noprefixroute
valid_lft forever preferred_lft forever
inet6 fe80::64d6:f397:92fd:1c0f/64 scope link noprefixroute
valid_lft forever preferred_lft forever
2-4:通过容器日记查看信息
bash
[root@localhost ~]# docker logs keepalived-master
2-5:出现错误关于keepalived执行后日志狂刷IPVS: Can't initialize ipvs: Protocol not available的问题
安装了keepalived+lvs,达到了高可用的负载均衡,但是今天再启用的时候发现keepalived不正常,通过 /var/log/messages 查看系统日志发现狂刷
2-6: 查看:ip_vs模块系统默认没有自动加载
bash
lsmod | grep ip_vs
2-7: 命令查看一下,如果没有任何输出则表示ip_vs模块并没有被内核加载,那必须手动加载一下
bash
modprobe ip_vs
modprobe ip_vs_wrr
2-8: 过程如下,如果要让系统开机加载此模块,在/etc/rc.local文件中添加上面两句,这样开机就能自动加载了
bash
[root@localhost ~]# modprobe ip_vs
[root@localhost ~]# modprobe ip_vs_wrr
[root@localhost ~]# lsmod | grep ip_vs
ip_vs_wrr 12697 0
ip_vs 145458 2 ip_vs_wrr
nf_conntrack 139264 7 ip_vs,nf_nat,nf_nat_ipv4,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_netlink,nf_conntrack_ipv4
libcrc32c 12644 4 xfs,ip_vs,nf_nat,nf_conntrack
[root@localhost ~]#
3:docker启动的配置文件目录在容器路径/container/service/keepalived/assets下
bash
/container/service/keepalived/assets # ls
README.md keepalived.conf notify.sh
/container/service/keepalived/assets # cat keepalived.conf
global_defs {
default_interface ens33
}
vrrp_instance VI_1 {
interface ens33
state BACKUP
virtual_router_id 51
priority 100
nopreempt #非抢占模式,如果设置降权重方式,必须要去掉该值或者注释,不然不会抢占
unicast_peer {
192.168.11.43
192.168.11.47
}
virtual_ipaddress {
192.168.11.40
}
authentication {
auth_type PASS
auth_pass d0cker
}
notify "/container/service/keepalived/assets/notify.sh"
}
3-1:MASTER的keepalived挂载配置文件keepalived.conf启动容器方式(重要主要:文件的权限必须设置为644,配置文件其他权限挂载keepalived启动不起来)
bash
docker run -d --name keepalived-master --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -v /opt/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf --privileged=true osixia/keepalived --loglevel debug --copy-service
bash
global_defs {
default_interface ens33
}
vrrp_script check_mysql {
script "/container/service/keepalived/assets/check_mysql.sh" #检测脚本文件
interval 5 #检测时间间隔,单位是秒
weight -10 #权重
#检测连续2次失败才算确定是真失败。会用weight减少优先级(1-255之间)
fall 2 #脚本结果导致的优先级变更,检测失败(脚本返回非0)则优先级 - 10
rise 1 #检测1次成功就算成功。但不修改优先级
}
vrrp_instance VI_1 {
interface ens33 #设置实例绑定的网卡
state BACKUP #设置实例初始状态,实际的MASTER和BACKUP是选举决定的
virtual_router_id 51 #同一实例下virtual_router_id必须相同
priority 100 #设置优先级,优先级高的会被竞选为Master
nopreempt #非抢占模式,如果设置降权重方式,必须要去掉该值或者注释,不然不会抢占
unicast_peer { #单播模式,设置对端ip
192.168.11.43
192.168.11.47
}
virtual_ipaddress { #设置VIP,可以设置多个
192.168.11.40
}
authentication { #设置认证
auth_type PASS #认证方式,支持PASS和AH
auth_pass d0cker #认证密码
}
track_script { #设置追踪脚本
check_mysql
}
notify "/container/service/keepalived/assets/notify.sh"
}
3-2:BACKUP的keepalived挂载配置文件keepalived.conf启动容器方式(重要主要:文件的权限必须设置为644,配置文件其他权限挂载keepalived启动不起来)
bash
docker run -d --name keepalived-backup --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -v /opt/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf --privileged=true osixia/keepalived --loglevel debug --copy-service
bash
global_defs {
default_interface ens33
}
vrrp_script check_mysql {
script "/container/service/keepalived/assets/check_mysql.sh" #检测脚本文件
interval 5 #检测时间间隔,注意这个时间必须比脚本执行的耗时时间长,不然运行都人为有问题,就会降权重,单位是秒
weight -10 #权重,priority值减去此值要小于备服务的priority值
#检测连续2次失败才算确定是真失败。会用weight减少优先级(1-255之间)
fall 2 #脚本结果导致的优先级变更,检测失败(脚本返回非0)
rise 1 #检测1次成功就算成功。但不修改优先级
}
vrrp_instance VI_1 {
interface ens33 #设置实例绑定的网卡
state BACKUP #设置实例初始状态,实际的MASTER和BACKUP是选举决定的
virtual_router_id 51 #同一实例下virtual_router_id必须相同
priority 50 #设置优先级,优先级高的会被竞选为Master
nopreempt #非抢占模式,如果设置降权重方式,必须要去掉该值或者注释,不然不会抢占
unicast_peer { #单播模式,设置对端ip
192.168.11.43
192.168.11.47
}
virtual_ipaddress { #设置VIP,可以设置多个
192.168.11.40
}
authentication { #设置认证
auth_type PASS #认证方式,支持PASS和AH
auth_pass d0cker #认证密码
}
track_script { #设置追踪脚本
check_mysql
}
notify "/container/service/keepalived/assets/notify.sh"
}
需要特别注意:nopreempt这个参数只能用于state为BACKUP的情况,所以在配置的时候要把master和backup的state都设置成BACKUP,这样才会实现keepalived的非抢占模式!
3-3:当MASTER出现故障自动切换BACKUP,当MASTER恢复,自动切换回MASTER,如果MASTER和BACKUP的KEEPALIVED_PRIORITY一样,需要手动才能切换
:vip 不释放问题,如果停止keepalived还不释放vip可使用命令删除如下
bash
ip addr del 192.168.11.40/32 dev ens33
3-4:mysql状态 check_mysql监听脚本,这个脚本是直接关闭keepalived(注意权限需要改为744)
bash
#!/bin/bash
#检测mysql是否启动了
if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #如果mysql没有启动就启动mysql
docker restart mysql #重启mysql
sleep 3 #注意设置这个时间一定要比interval小,不然一直认为执行脚本失败,单位是秒
if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #mysql重启失败,则停掉keepalived服务,进行VIP转移
systemctl stop keepalived #非容器部署
kill 1 #容器部署
fi
fi
3-5:mysql状态 check_mysql监听脚本,这个脚本是返回值,可以检测几次,和动态减去权重(注意权限需要改为744)
bash
#!/bin/bash
#检测mysql是否启动了
if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #如果mysql没有启动就启动mysql
docker restart mysql #重启mysql
sleep 3 #注意设置这个时间一定要比interval小,不然一直认为执行脚本失败,单位是秒
if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #mysql重启失败,返回 0
exit 0
else
exit 1 #mysql重启存在,返回 1
fi
fi
3-6:mysql状态 chk_mysql_port监听脚本,这个脚本是直接关闭keepalived(注意权限需要改为744)
bash
#!/bin/bash
counter=$(netstat -na|grep "LISTEN"|grep "3306"|wc -l)
if [ "${counter}" -eq 0 ]; then
systemctl stop keepalived #非容器部署
kill 1 #容器部署
fi
3-7:挂载配置文件和挂载脚本的docker启动方式
bash
docker run -d --name keepalived-master --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -v /opt/keepalived/chk_mysql_port.sh:/container/service/keepalived/assets/chk_mysql_port.sh -v /opt/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf --privileged=true osixia/keepalived --loglevel debug --copy-service
3-8:nginx状态监听脚本
bash
#!/bin/bash
#检测nginx是否启动了
if [ "$(ps -ef | grep "nginx: master process"| grep -v grep )" == "" ];then #如果nginx没有启动就启动nginx
docker restart nginx #重启nginx
sleep 3 #单位是秒
if [ "$(ps -ef | grep "nginx: master process"| grep -v grep )" == "" ];then #nginx重启失败,则停掉keepalived服务,进行VIP转移
systemctl stop keepalived #非容器部署
kill 1 #容器部署
fi
fi
3-9:可以通过docker logs -f keepalived-master实时查看日记,也可以看到降权和复权的过程
bash
Thu Mar 14 11:48:22 2024: Script `check_mysql_port` now returning 1
Thu Mar 14 11:48:22 2024: VRRP_Script(check_mysql_port) failed (exited with status 1)
Thu Mar 14 11:48:22 2024: (VI_1) Changing effective priority from 100 to 40
Thu Mar 14 11:48:25 2024: (VI_1) Receive advertisement timeout
Thu Mar 14 11:48:25 2024: (VI_1) Entering MASTER STATE
Thu Mar 14 11:48:25 2024: (VI_1) setting VIPs.
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: (VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
I'm the MASTER! Whup whup.
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: (VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:43 2024: Interface veth3a6fe0a added
Thu Mar 14 11:48:43 2024: Interface veth8c4218d added
Thu Mar 14 11:48:43 2024: Interface veth3a6fe0a deleted
Thu Mar 14 11:48:46 2024: Script `check_mysql_port` now returning 0
Thu Mar 14 11:48:46 2024: VRRP_Script(check_mysql_port) succeeded
Thu Mar 14 11:48:46 2024: (VI_1) Changing effective priority from 40 to 100
Thu Mar 14 11:49:45 2024: Interface veth3a6fe0a added
Thu Mar 14 11:49:45 2024: Interface veth3a6fe0a deleted
Thu Mar 14 11:49:45 2024: Interface veth8c4218d deleted
Thu Mar 14 11:49:45 2024: Unknown interface deleted
Thu Mar 14 11:49:50 2024: Script `check_mysql_port` now returning 1
Thu Mar 14 11:49:54 2024: VRRP_Script(check_mysql_port) failed (exited with status 1)
Thu Mar 14 11:49:54 2024: (VI_1) Changing effective priority from 100 to 40
4:负载均衡的keepalived.conf配置文件讲解
go
global_defs { #全局定义部分
notification_email { #设置报警邮件地址,可设置多个
acassen@firewall.loc #接收通知的邮件地址
}
notification_email_from test0@163.com #设置 发送邮件通知的地址
smtp_server smtp.163.com #设置 smtp server 地址,可是ip或域名.可选端口号 (默认25)
smtp_connect_timeout 30 #设置 连接 smtp server的超时时间
router_id LVS_DEVEL #主机标识,用于邮件通知
vrrp_skip_check_adv_addr
vrrp_strict #严格执行VRRP协议规范,此模式不支持节点单播
vrrp_garp_interval 0
vrrp_gna_interval 0
script_user keepalived_script #指定运行脚本的用户名和组。默认使用用户的默认组。如未指定,默认为keepalived_script 用户,如无此用户,则使用root
enable_script_security #如果路径为非root可写,不要配置脚本为root用户执行。
}
vrrp_script chk_nginx_service { #VRRP 脚本声明
script "/etc/keepalived/chk_nginx.sh" #周期性执行的脚本
interval 3 #运行脚本的间隔时间,秒
weight -20 #权重,priority值减去此值要小于备服务的priority值
fall 3 #检测几次失败才为失败,整数
rise 2 #检测几次状态为正常的,才确认正常,整数
user keepalived_script #执行脚本的用户或组
}
vrrp_instance VI_1 { #vrrp 实例部分定义,VI_1自定义名称
state MASTER #指定 keepalived 的角色,必须大写 可选值:MASTER|BACKUP
interface ens33 #网卡设置,lvs需要绑定在网卡上,realserver绑定在回环口。区别:lvs对访问为外,realserver为内不易暴露本机信息
virtual_router_id 51 #虚拟路由标识,是一个数字,同一个vrrp 实例使用唯一的标识,MASTER和BACKUP 的 同一个 vrrp_instance 下 这个标识必须保持一致
priority 100 #定义优先级,数字越大,优先级越高。
advert_int 1 #设定 MASTER 与 BACKUP 负载均衡之间同步检查的时间间隔,单位为秒,两个节点设置必须一样
authentication { #设置验证类型和密码,两个节点必须一致
auth_type PASS
auth_pass 1111
}
virtual_ipaddress { #设置虚拟IP地址,可以设置多个虚拟IP地址,每行一个
192.168.119.130
}
track_script { #脚本监控状态
chk_nginx_service #可加权重,但会覆盖声明的脚本权重值。chk_nginx_service weight -20
}
notify_master "/etc/keepalived/start_haproxy.sh start" #当前节点成为master时,通知脚本执行任务
notify_backup "/etc/keepalived/start_haproxy.sh stop" #当前节点成为backup时,通知脚本执行任务
notify_fault "/etc/keepalived/start_haproxy.sh stop" #当当前节点出现故障,执行的任务;
}
## 虚拟服务器virtual_server定义块 ,虚拟服务器定义是keepalived框架最重要的项目了,是keepalived.conf必不可少的部分。 该部分是用来管理LVS的,是实现keepalive和LVS相结合的模块。
virtual_server 192.168.119.130 80 { #定义RealServer对应的VIP及服务端口,IP和端口之间用空格隔开
delay_loop 6 #每隔6秒查询realserver状态
lb_algo rr #后端调试算法(load balancing algorithm)
lb_kind DR #LVS调度类型NAT/DR/TUN
#persistence_timeout 60 同一IP的连接60秒内被分配到同一台realserver
protocol TCP #用TCP协议检查realserver状态
real_server 192.168.119.120 80 {
weight 1 #权重,最大越高,lvs就越优先访问
TCP_CHECK { #keepalived的健康检查方式HTTP_GET | SSL_GET | TCP_CHECK | SMTP_CHECK | MISC
connect_timeout 10 #10秒无响应超时
retry 3 #重连次数3次
delay_before_retry 3 #重连间隔时间
connect_port 80 #健康检查realserver的端口
}
}
real_server 192.168.119.121 80 {
weight 1 #权重,最大越高,lvs就越优先访问
TCP_CHECK { #keepalived的健康检查方式HTTP_GET | SSL_GET | TCP_CHECK | SMTP_CHECK | MISC
connect_timeout 10 #10秒无响应超时
retry 3 #重连次数3次
delay_before_retry 3 #重连间隔时间
connect_port 80 #健康检查realserver的端口
}
}
}
4-1:虚拟服务器转发
bash
! Configuration File for keepalived
global_defs {
router_id db01
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 51
priority 100
nopreempt
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.132.40
}
}
virtual_server 192.168.11.40 3306 { #定义了一个虚拟服务器,监听 IP 地址为 192.168.11.40,端口为 3306(MySQL 默认端口)
delay_loop 6 #设置检查间隔为 6 秒
persistence_timeout 50 #设置持久性失败计数器超时时间为 50
protocol TCP
real_server 192.168.11.43 3306 { #定义了一个真实服务器,即要监控的 MySQL 服务所在的服务器
notify_down /etc/keepalived/kill_keepalived.sh #当检测到真实服务器不可用时,执行 /etc/keepalived/kill_keepalived.sh 脚本,通常用于触发切换操作
TCP_CHECK {
connect_timeout 3 #连接超时时间为 3 秒
nb_get_retry 3 #最大重试次数为 3 次
connect_port 3306 #连接的端口为 3306
delay_before_retry 3 #重试前等待 3 秒
}
}
}