Featured image of post keepalived + lvs实践
linux

keepalived + lvs实践

keepalived 部署

环境介绍

角色 操作系统 IP VIP 主要软件
keepalived主节点 RockyLinux9.3 172.31.31.132 172.31.31.231 keepalived 2.2.8-3
keepalived从节点 RockyLinux9.3 172.31.31.133 172.31.31.231 keepalived 2.2.8-3
real server1 RockyLinux9.3 172.31.31.134 - nginx 1.20.1
real server2 RockyLinux9.3 172.31.31.135 - nginx 1.20.1

keepalived + lvs

安装程序

1

dnf install -y keepalived ipvsadm

外部配置

启用ipvs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

$ cat > /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

$ systemctl restart systemd-modules-load.service

$ lsmod | grep ip_vs
# ip_vs_sh               16384  0
# ip_vs_wrr              16384  0
# ip_vs_rr               16384  0
# ip_vs                 188416  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
# nf_conntrack          176128  3 nf_nat,nft_ct,ip_vs
# nf_defrag_ipv6         24576  2 nf_conntrack,ip_vs
# libcrc32c              16384  5 nf_conntrack,nf_nat,nf_tables,xfs,ip_vs$ lsmod | grep nf_conntrack

程序配置

配置详情

这是keepalived默认自带的示例：

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

! Configuration File for keepalived

global_defs {
   # 邮件通知信息
   notification_email {  
     # 定义收件人
     acassen@firewall.loc  
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   # 定义发件人
   notification_email_from Alexandre.Cassen@firewall.loc
   # SMTP服务器地址
   smtp_server 192.168.200.1
   # SMTP连接超时时间 
   smtp_connect_timeout 30  
   # 本节点的Router ID
   router_id LVS_DEVEL
   # 跳过VRRP广播地址检查
   vrrp_skip_check_adv_addr
   # 严格遵守VRRP协议
   vrr_strict
   # VRRP广播周期间隔
   vrrp_garp_interval 0
   # VRRP邻居广播周期间隔
   vrrp_gna_interval 0 
}

# VRRP实例配置
vrrp_instance VI_1 {
    # 定义初始状态，可以是MASTER或者BACKUP
    state MASTER 
    # 绑定接口
    interface eth0
    # 虚拟路由ID，如果是一组虚拟路由就定义一个ID，如果是多组就要定义多个，而且这个虚拟
    # ID还是虚拟MAC最后一段地址的信息，取值范围0-255
    virtual_router_id 51
    # 优先级，如果是Master，这里的优先级就需要定义的比其他的高
    priority 100
    # 通告频率，单位为秒
    advert_int 1
    # 认证配置
    authentication {
        # 认证方式
        auth_type PASS
        # 认证密码
        auth_pass 1111
    }
    # 虚拟IP地址
    virtual_ipaddress {
        192.168.200.16
        192.168.200.17
        192.168.200.18
    }
}

# 虚拟服务器 1
virtual_server 192.168.200.100 443 {
    # 检测间隔
    delay_loop 6
    # 负载均衡算法 可用值为：rr|wrr|lc|wlc|lblc|sh|dh
    lb_algo rr 
    # LB的模式，NAT|DR|TUN
    lb_kind NAT
    # 持久连接时间
    persistence_timeout 50
    # 协议 TCP|UDP
    protocol TCP

    # 真实服务器1
    real_server 192.168.201.100 443 {
        # 权重
        weight 1
        # 健康检查，MSIC_CHECK|SMTP_CHEKC|TCP_CHECK|SSL_GET|HTTP_GET这些都是针对应用服务器做健康检查的方法
        SSL_GET {
            url {
              # 检查路径
              path / 
              # 响应内容md5
              digest ff20ad2481f97b1754ef3e12ecd3a9cc
            }
            url {
              path /mrtg/
              digest 9b3a0c85a887a256d6939da88aabd8cd
            }
            # 连接超时时间
            connect_timeout 3
            # 重试次数
            retry 3
            # 重试间隔
            delay_before_retry 3
        }
    }
}

# 虚拟服务器 2
virtual_server 10.10.10.2 1358 {
    delay_loop 6
    lb_algo rr
    lb_kind NAT
    persistence_timeout 50
    protocol TCP

    # 备用服务器
    sorry_server 192.168.200.200 1358 

    # 真实服务器1
    real_server 192.168.200.2 1358 {
        weight 1
        HTTP_GET {
            url {
              path /testurl/test.jsp
              digest 640205b7b0fc66c1ea91c463fac6334d
            }
            url {
              path /testurl2/test.jsp
              digest 640205b7b0fc66c1ea91c463fac6334d
            }
            url {
              path /testurl3/test.jsp
              digest 640205b7b0fc66c1ea91c463fac6334d
            }
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }

    # 真实服务器2
    real_server 192.168.200.3 1358 {
        weight 1
        HTTP_GET {
            url {
              path /testurl/test.jsp
              digest 640205b7b0fc66c1ea91c463fac6334c
            }
            url {
              path /testurl2/test.jsp
              digest 640205b7b0fc66c1ea91c463fac6334c
            }
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
}

我的配置

主节点配置：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

# Master
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict #我的机器启用了ipv6这个项不能开启
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER
    interface ens160
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1314
    }
    virtual_ipaddress {
        172.31.31.231
    }
}


virtual_server 172.31.31.231 80 {
    delay_loop 1
    lb_algo rr
    lb_kind DR
#    persistence_timeout 1
    protocol TCP

    sorry_server 172.15.110.34 9010

    real_server 172.31.31.134 80 {
        weight 1
        HTTP_GET {
            url {
              path /index.html
              status_code 200
            }
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }

    real_server 172.31.31.135 80 {
        weight 1
        HTTP_GET {
            url {
              path /index.html
              status_code 200
            }
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
}

从节点配置：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

# backup
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens160
    virtual_router_id 51
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1314
    }
    virtual_ipaddress {
        172.31.31.231
    }
}


virtual_server 172.31.31.231 80 {
    delay_loop 1
    lb_algo rr
    lb_kind DR
#    persistence_timeout 1
    protocol TCP

    sorry_server 172.15.110.34 9010

    real_server 172.31.31.134 80 {
        weight 1
        HTTP_GET {
            url {
              path /index.html
              status_code 200
            }
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }

    real_server 172.31.31.135 80 {
        weight 1
        HTTP_GET {
            url {
              path /index.html
              status_code 200
            }
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
}

验证

环境配置验证

主节点验证VIP分配与否

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

$ ip addr
#1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#    inet 127.0.0.1/8 scope host lo
#       valid_lft forever preferred_lft forever
#    inet6 ::1/128 scope host
#       valid_lft forever preferred_lft forever
#2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
#    link/ether 00:0c:29:05:70:4f brd ff:ff:ff:ff:ff:ff
#    altname enp3s0
#    inet 172.31.31.132/24 brd 172.31.31.255 scope global dynamic noprefixroute ens160
#       valid_lft 859924sec preferred_lft 859924sec
#    inet 172.31.31.231/32 scope global ens160
#       valid_lft forever preferred_lft forever
#    inet6 fe80::20c:29ff:fe05:704f/64 scope link noprefixroute
#       valid_lft forever preferred_lft forever
#3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
#    link/ipip 0.0.0.0 brd 0.0.0.0

检查路由规则

1
2
3
4

$ ip route
#default via 172.31.31.2 dev ens160 proto dhcp src 172.31.31.132 metric 100
#127.0.0.0/8 dev lo proto kernel scope link src 127.0.0.1 metric 30
#172.31.31.0/24 dev ens160 proto kernel scope link src 172.31.31.132 metric 100

检查ipvs规则

1
2
3
4
5
6
7

$ ipvsadm -ln
#IP Virtual Server version 1.2.1 (size=4096)
#Prot LocalAddress:Port Scheduler Flags
#  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
#TCP  172.31.31.231:80 rr
#  -> 172.31.31.134:80             Route   1      0          0
#  -> 172.31.31.135:80             Route   1      0          0

功能验证

主节点运行中，启动从节点日志（此时主节点没有日志）

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

Dec 25 16:24:35 localhost.localdomain systemd[1]: Starting LVS and VRRP High Availability Monitor...
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Starting Keepalived v2.2.8 (04/04,2023), git commit v2.2.7-154-g292b299e+
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Running on Linux 5.14.0-362.8.1.el9_3.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Nov 8 17:36:32 UTC 2023 (built for Linux 5.14.0)
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Command line: '/usr/sbin/keepalived' '--dont-fork' '-D'
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Opening file '/etc/keepalived/keepalived.conf'.
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Configuration file /etc/keepalived/keepalived.conf
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: (Line 7) WARNING - number '0' outside range [0.000001, 4294.967295]
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: (Line 7) vrrp_garp_interval '0' is invalid
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: (Line 8) WARNING - number '0' outside range [0.000001, 4294.967295]
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: (Line 8) vrrp_gna_interval '0' is invalid
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: NOTICE: setting config option max_auto_priority should result in better keepalived performance
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Starting Healthcheck child process, pid=14876
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Starting VRRP child process, pid=14877
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: Registering Kernel netlink reflector
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: Registering Kernel netlink command channel
Dec 25 16:24:35 localhost.localdomain Keepalived_healthcheckers[14876]: WARNING - virtual server [172.31.31.231]:tcp:80 and sorry server ports don't match - resetting
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: Assigned address 172.31.31.133 for interface ens160
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: Assigned address fe80::20c:29ff:fe2e:5b54 for interface ens160
Dec 25 16:24:35 localhost.localdomain Keepalived_healthcheckers[14876]: Initializing ipvs
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: Registering gratuitous ARP shared channel
Dec 25 16:24:35 localhost.localdomain Keepalived_healthcheckers[14876]: Gained quorum 1+0=1 <= 11 for VS [172.31.31.231]:tcp:80
Dec 25 16:24:35 localhost.localdomain Keepalived_healthcheckers[14876]: Activating healthchecker for service [172.31.31.134]:tcp:80 for VS [172.31.31.231]:tcp:80
Dec 25 16:24:35 localhost.localdomain Keepalived_healthcheckers[14876]: Activating healthchecker for service [172.31.31.135]:tcp:80 for VS [172.31.31.231]:tcp:80
Dec 25 16:24:35 localhost.localdomain Keepalived[14875]: Startup complete
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) removing VIPs.
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Entering BACKUP STATE (init)
Dec 25 16:24:35 localhost.localdomain systemd[1]: Started LVS and VRRP High Availability Monitor.
Dec 25 16:24:35 localhost.localdomain Keepalived_vrrp[14877]: VRRP sockpool: [ifindex(  2), family(IPv4), proto(112), fd(12,13) multicast, address(224.0.0.18)]
Dec 25 16:24:36 localhost.localdomain Keepalived_healthcheckers[14876]: Remote Web server [172.31.31.134]:tcp:80 succeed on service.
Dec 25 16:24:36 localhost.localdomain Keepalived_healthcheckers[14876]: Remote Web server [172.31.31.135]:tcp:80 succeed on service.
Dec 25 16:24:50 localhost.localdomain systemd[1]: systemd-hostnamed.service: Deactivated successfully.

停止主节点，从节点升级成主节点，并获得VIP

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

Dec 25 16:26:05 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Backup received priority 0 advertisement
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Receive advertisement timeout
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Entering MASTER STATE
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) setting VIPs.
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Sending/queueing gratuitous ARPs on ens160 for 172.31.31.231
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:06 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:11 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Sending/queueing gratuitous ARPs on ens160 for 172.31.31.231
Dec 25 16:26:11 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:11 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:11 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:11 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231
Dec 25 16:26:11 localhost.localdomain Keepalived_vrrp[14877]: Sending gratuitous ARP on ens160 for 172.31.31.231

启动主节点，从节点移除VIP

1
2
3

Dec 25 16:26:38 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Master received advert from 172.31.31.132 with higher priority 100, ours 99
Dec 25 16:26:38 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) Entering BACKUP STATE
Dec 25 16:26:38 localhost.localdomain Keepalived_vrrp[14877]: (VI_1) removing VIPs.

RealServer服务器

realserver 服务器需要修改 ARP 参数并在lo接口上添加 VIP，主要原因有:

  • arp_ignore 和 arp_announce:这些设置可以防止 realserver 对 VIP 地址的 ARP 请求做出响应。这样可以避免 realserver 与 keepalived VIP 之间的 ARP 冲突。

  • 在loopback添加VIP: realserver需要配置VIP，以便它可以接受发往该IP的数据包。在loopback上添加可以避免与主网卡的冲突。

  • 通过loopback路由: 这可以确保 VIP 连接的返回流量通过loopback上的VIP返回，而不是主网卡。

也可以用这个脚本：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

#!/bin/bash

vip=$1
dev=lo
case $2 in
start)
    sysctl -w net.ipv4.conf.all.arp_ignore=1
    sysctl -w net.ipv4.conf.lo.arp_ignore=1
    sysctl -w net.ipv4.conf.all.arp_announce=2
    sysctl -w net.ipv4.conf.lo.arp_announce=2
    ip addr add $vip/32 dev $dev brd $vip
    ip route add $vip dev lo
    nmcli d connect lo
    echo "The Real Server is Ready!"
    ;;
stop)
    sysctl -w net.ipv4.conf.all.arp_ignore=0
    sysctl -w net.ipv4.conf.lo.arp_ignore=0
    sysctl -w net.ipv4.conf.all.arp_announce=0
    sysctl -w net.ipv4.conf.lo.arp_announce=0
    ip add del $vip/32 dev $dev
    ip route del $vip dev lo
    nmcli d disconnect lo
    echo "The Real Server is Stopped!"
    ;;
*)
    echo "Usage: "
    echo "   start with: $(basename $0) [your_vip_address] start "
    echo "   stop  with: $(basename $0) [your_vip_address] stop"
    exit 1
    ;;
esac

每个后端服务器执行这个脚本，比如这个脚本叫setRS.sh，VIP是172.31.31.231

1

$ ./setRS.sh 172.31.31.231 start

验证

环境配置验证

验证内核参数已经正确修改

1
2
3
4
5

$ sysctl -a | grep arp_ | grep -v 0
#net.ipv4.conf.all.arp_announce = 2
#net.ipv4.conf.all.arp_ignore = 1
#net.ipv4.conf.lo.arp_announce = 2
#net.ipv4.conf.lo.arp_ignore = 1

验证lo已经设置为connected

1
2
3
4

$ nmcli d
#DEVICE  TYPE      STATE      CONNECTION
#ens160  ethernet  connected  ens160
#lo      loopback  connected  lo

ip已经添加到lo

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

$ ip addr
#1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
#    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
#    inet 127.0.0.1/8 scope host lo
#       valid_lft forever preferred_lft forever
#    inet 172.31.31.231/32 scope global lo
#       valid_lft forever preferred_lft forever
#    inet6 ::1/128 scope host
#       valid_lft forever preferred_lft forever
#2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
#    link/ether 00:0c:29:e1:ec:80 brd ff:ff:ff:ff:ff:ff
#    altname enp3s0
#    inet 172.31.31.134/24 brd 172.31.31.255 scope global dynamic noprefixroute ens160
#       valid_lft 859628sec preferred_lft 859628sec
#    inet6 fe80::20c:29ff:fee1:ec80/64 scope link noprefixroute
#       valid_lft forever preferred_lft forever

功能验证

停止/启动一个realserver，keepalived日志（可以正确上线和下线realserver）：

1
2
3
4
5
6

Dec 25 16:21:58 localhost.localdomain Keepalived_healthcheckers[8088]: HTTP_CHECK on service [172.31.31.135]:tcp:80 failed after 3 retries.
Dec 25 16:21:58 localhost.localdomain Keepalived_healthcheckers[8088]: Removing service [172.31.31.135]:tcp:80 from VS [172.31.31.231]:tcp:80
Dec 25 16:22:34 localhost.localdomain Keepalived_healthcheckers[8088]: HTTP status code success to [172.31.31.135]:tcp:80 url(/index.html)
Dec 25 16:22:35 localhost.localdomain Keepalived_healthcheckers[8088]: HTTP status code success to [172.31.31.135]:tcp:80 url(/index.html)
Dec 25 16:22:35 localhost.localdomain Keepalived_healthcheckers[8088]: Remote Web server [172.31.31.135]:tcp:80 succeed on service.
Dec 25 16:22:35 localhost.localdomain Keepalived_healthcheckers[8088]: Adding service [172.31.31.135]:tcp:80 to VS [172.31.31.231]:tcp:80