lvs DR模式调试

DS配置：

# cat /etc/keepalived_docker/keepalived.conf 
! Configuration File for keepalived
global_defs {
  router_id LVS_70          # 设置lvs的id，在一个网络内应该是唯一的
}

vrrp_instance VI_70 {
  state MASTER            # 两个 DS，一个为 MASTER，一个为 BACKUP
  interface ens33         # 当前 IP 对应的网络接口
  virtual_router_id 70    # 虚拟路由 ID(0-255)，在一个 VRRP 实例中主备服务器 ID 必须一样
  priority 100            # 优先级值设定：MASTER 要比 BACKUP 的值大
  advert_int 1            # 通告时间间隔：单位秒，主备要一致
  authentication {        # 认证机制，主从节点保持一致即可
    auth_type PASS
    auth_pass 1111
  }
  virtual_ipaddress {
    192.168.137.70        # VIP，可配置多个
  }
}

# LB 配置，定义对外提供服务的LVS的VIP以及port
virtual_server 192.168.137.70 80 {
  delay_loop 3                    # 设置健康状态检查时间
  lb_algo wlc                     # 设置负载调度的算法，wrr/rr等
  lb_kind DR                      # 这里测试用了 Direct Route 模式
  persistence_timeout 600         # 持久连接超时时间即会话保持时间，这段时间内同一ip发起的请求将被转发到同一个 RS
  protocol TCP
  virtualhost LVS

  real_server 192.168.137.3 80 {
    weight 1
    TCP_CHECK {
      connect_timeout 3
      retry 3                     # 旧版本为 nb_get_retry
      delay_before_retry 2
      connect_port 80
    }
  }

  real_server 192.168.137.4 80 {
    weight 1
    TCP_CHECK {
      connect_timeout 3
      retry 3                     # 旧版本为 nb_get_retry
      delay_before_retry 2
      connect_port 80
    }
  }
}

DS开通ip转发：

#vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
sysctl -p

DS网卡信息：

root@el7 lvs\]# ip a 1: lo: \ mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: \ mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:af:ed:5f brd ff:ff:ff:ff:ff:ff inet 192.168.137.2/24 brd 192.168.137.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet 192.168.137.60/32 scope global ens33 valid_lft forever preferred_lft forever inet 192.168.137.70/32 scope global ens33 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:feaf:ed5f/64 scope link valid_lft forever preferred_lft forever

RS脚本：

# cat lvs-rs.sh 
#!/bin/bash
VIP=192.168.137.70
#/etc/rc.d/init.d/functions
case "$1" in
  start)
    echo "Start Real Server ..."
    ifconfig lo:0 $VIP netmask 255.255.255.255 broadcast $VIP up
    route add -host $VIP dev lo:0
    #如果多个vip继续类似添加：
    #ifconfig lo:2 $VIP2 netmask 255.255.255.255 broadcast $VIP2 up
    #route add -host $VIP2 dev lo:2
    echo "1" > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "2" > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "1" > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo "2" > /proc/sys/net/ipv4/conf/all/arp_announce
    sysctl -p > /dev/null 2>&1
    systemctl stop firewalld
    echo "Real Server Started."
    ;;
  stop)
    echo "Stop Real Server ..."
    ifconfig lo:0 down
    #如果多个vip继续类似添加：
    #ifconfig lo:2 down
    route del $VIP >/dev/null 2>&1
    echo "0" > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "0" > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "0" > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo "0" > /proc/sys/net/ipv4/conf/all/arp_announce
    echo "Real Server Stopped."
    ;;
  *)
    echo "Usage: $0 {start|stop}"
    exit 1
esac
exit 0

# 添加VIP，修改内核参数（实现对内可见、对外隐藏），注意掩码必须是255.255.255.255
# 因为VIP是绑在环回网卡上（回环网卡离内核更近，数据包优先匹配）的，如果掩码不是4个255，则数据包在返回的时候匹配上了回环网卡，匹配不上真实网关，数据包发送不出去
#RS、DS 有同样的vip，同一个网段中拥有两个vip，客户端在网关发送arp广播需找vip时需要让RS不接受响应。
#arp_ignore 设置为1，意味着当别人的arp请求过来的时候，如果接收的设备没有这个ip，就不做出响应(这个ip在lo上，lo不是接收设备的进口)
#使用最好的ip来回应，什么是最好的ip？同一个网段内子网掩码最长的

RS网卡信息：

root@el8 lvs\]# ip a 1: lo: \ mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet 192.168.137.60/32 brd 192.168.137.60 scope global lo:0 valid_lft forever preferred_lft forever inet 192.168.137.70/32 brd 192.168.137.70 scope global lo:1 valid_lft forever preferred_lft forever inet 39.134.209.254/32 brd 39.134.209.254 scope global lo:254 valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: \ mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:3f:10:fe brd ff:ff:ff:ff:ff:ff inet 192.168.137.3/24 brd 192.168.137.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe3f:10fe/64 scope link valid_lft forever preferred_lft forever

查看lvs状态：

ipvsadm -Ln --persistent-conn
echo
ipvsadm -Ln --stats

如：

root@el7 lvs\]# sh watch-ipvsadm.sh IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Weight PersistConn ActiveConn InActConn -\> RemoteAddress:Port TCP 192.168.137.60:80 wlc persistent 600 -\> 192.168.137.3:80 1 0 0 0 -\> 192.168.137.4:80 1 0 0 0 TCP 192.168.137.70:80 wlc persistent 600 -\> 192.168.137.3:80 1 0 0 0 -\> 192.168.137.4:80 1 0 0 0 IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Conns InPkts OutPkts InBytes OutBytes -\> RemoteAddress:Port TCP 192.168.137.60:80 2 216 0 9006 0 -\> 192.168.137.3:80 2 216 0 9006 0 -\> 192.168.137.4:80 0 0 0 0 0 TCP 192.168.137.70:80 9 61 0 3097 0 -\> 192.168.137.3:80 9 61 0 3097 0 -\> 192.168.137.4:80 0 0 0 0 0

ipvsadm管理命令：

#手工添加vip和dr规则
ip addr add $vip dev bond0
ipvsadm -A -t $vip:80 -s wrr
ipvsadm -a -t $vip:80 -r $rs1:80 -g
ipvsadm -a -t $vip:80 -r $rs2:80 -g

#将某个RS的权重调为0：
ipvsadm -e -t 192.168.137.70:80 -r 192.168.137.3 -g -w 0

#删除某个vip
ipvsadm -D -t 192.168.137.70:80

客户端测试：

curl -vo /dev/null "http://mytest.com/test.mp3" -H "User-Agent:mytest" -m 600 -x 192.168.137.60:80

vrrp抓包：

tcpdump -i ens33 vrrp -n

lvs tcpdump抓包：

tcpdump -i any -e -n -nn -s 0 '(host $lvs_vip or host $client_ip) and tcp port $service_port'

如：

tcpdump -i any -e -n -nn -s 0 '(host 192.168.137.70 or host 192.168.137.1) and tcp port 80'

错误处理：

问题1：IPVS (cmd 1159,errno 2): No such file or directory

**解决：**ipvs cmd 1160 is IP_VS_SO_SET_DELDEST and cmd 1159 is IP_VS_SO_SET_ADDDEST so it appears to be attempting to delete the failed real server and then add the real server.

--end--