# Install keepalived
# Ubuntu
apt-get install keepalived ipvsadm
# CentOS
yum install keepalived ipvsadm

# update iptables
vim /etc/sysconfig/iptables

# For keepalived:
# allow vrrp 
-A INPUT -p vrrp -j ACCEPT
-A INPUT -p igmp -j ACCEPT
# allow multicast
-A INPUT -d 224.0.0.18 -j ACCEPT

# reload iptables
service iptables reload

# open ip_forward
echo "1" > /proc/sys/net/ipv4/ip_forward
# edit sysctl.conf
vi /etc/sysctl.conf
net.ipv4.ip_forward = 1

sysctl -p

# keepalived for lvs-dr
vim /etc/keepalived/keepalived.conf

vrrp_sync_group GOP {
    group {
        VI_PRI_CONNECT
        VI_PRI_AUTH
    }
}

vrrp_instance VI_PRI_CONNECT {
    state BACKUP
    interface bond0
    virtual_router_id 128
    priority 100
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.65.32.28/23 dev bond0
    }
}

virtual_server 10.65.32.28 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP

    real_server 10.65.32.13 80 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 80
        }
    }
    real_server 10.65.32.14 80 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 80
        }
    }
}

virtual_server 10.65.32.28 443 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP

    real_server 10.65.32.13 443 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 443
        }
    }
    real_server 10.65.32.14 80 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 443
        }
    }
}

vrrp_instance VI_PRI_AUTH {
    state BACKUP
    interface bond0
    virtual_router_id 129
    priority 100
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.65.32.29/23 dev bond0
    }
}

virtual_server 10.65.32.29 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP

    real_server 10.65.32.22 80 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 80
        }
    }
    real_server 110.65.32.23 80 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 80
        }
    }
}

virtual_server 10.65.32.29 443 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP

    real_server 10.65.32.22 443 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 443
        }
    }
    real_server 110.65.32.23 443 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
                connect_port 443
        }
    }
}

# enable and start keepalived
systemctl start keepalived
systemctl enable keepalived
watch ipvsadm -L -n --stats

# add TYPE=Loopback
echo "TYPE=Loopback" >> /etc/sysconfig/network-scripts/ifcfg-lo
# add ifcfg-lo:0
cat > /etc/sysconfig/network-scripts/ifcfg-lo:0 << EOF
DEVICE=lo:0
IPADDR=10.65.32.28
NETMASK=255.255.255.255
ONBOOT=yes
EOF

# ifup lo:0
ifup lo:0

# add real_start
cat > /root/real_start.sh << EOF
#!/bin/bash
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
EOF

# chmod 755
chmod 755 /root/real_start.sh

# add real.service
cat > /usr/lib/systemd/system/real.service << EOF
[Unit]
Description=autostart lvs real
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
ExecStart=/root/real_start.sh

[Install]
WantedBy=multi-user.target
EOF

# enable service
systemctl enable real.service


# lvs real server example
vim /root/lvs_real.sh

#!/bin/bash
### BEGIN INIT INFO
# Provides:
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start realserver
# Description:       Start realserver
### END INIT INFO

# change the VIP to proper value
VIP=10.65.32.28

case "$1" in
    start)

    echo "Start REAL Server"
    /sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
    echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce

    ;;

    stop)

    /sbin/ifconfig lo:0 down
    echo "Stop REAL Server"
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce

    ;;

    restart)

    $0 stop
    $0 start

    ;;

    *)

    echo "Usage: $0 {start|stop}"
    exit 1

    ;;
esac

exit 0

# 关于 3 中模式的参数 
[packet-forwarding-method]
       -g, --gatewaying  Use gatewaying (direct routing). This is the default.
       -i, --ipip  Use ipip encapsulation (tunneling).
       -m, --masquerading  Use masquerading (network access translation, or NAT).
       Note:  Regardless of the packet-forwarding mechanism specified, real servers for addresses for which there are interfaces on the local node will  be  use  the
       local  forwarding  method, then packets for the servers will be passed to upper layer on the local node. This cannot be specified by ipvsadm, rather it set by
       the kernel as real servers are added or modified.

# ipvsadm 命令行混配 
/sbin/ifconfig tunl0 10.10.36.11 broadcast 10.10.36.11 netmask 255.255.255.255 up
/sbin/route add -host 10.10.36.11 dev tunl0
/sbin/ipvsadm -At 10.10.36.11:80 -s rr
/sbin/ipvsadm -at 10.10.36.11:80 -r 10.10.36.4:80 -g -w 1
/sbin/ipvsadm -at 10.10.36.11:80 -r 10.10.36.7:80 -i -w 1

# keepalived 混配 
vrrp_sync_group GOP {
    group {
        VI_PRI_AUTH
    }
}

vrrp_instance VI_PRI_AUTH {
    state BACKUP
    interface em1
    virtual_router_id 11
    priority 100
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.10.36.11/23 dev em1
    }
}

virtual_server 10.10.36.11 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP

    real_server 10.10.36.4 80 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
        }
    }
}

virtual_server 10.10.36.11 80 {
    delay_loop 6
    lb_algo rr
    lb_kind TUN
    protocol TCP

    real_server 10.10.36.7 80 {
        weight 100
        TCP_CHECK {
                connect_timeout 3
                retry 3
                delay_before_retry 3
        }
    }
}

# 检查结果可用 
[root@d126027 wangao]# for i in {1..100}; do curl 10.10.36.11; sleep 0.5; done
rs2
rs1
rs2
rs1
rs2

[root@d126009 keepalived]# ipvsadm -Ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
  -> RemoteAddress:Port
TCP  10.10.36.11:80                    100      700        0    36700        0
  -> 10.10.36.4:80                      50      350        0    18350        0
  -> 10.10.36.7:80                      50      350        0    18350        0

[root@d126009 wangao]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.10.36.11:80 rr
  -> 10.10.36.4:80                Route   100    0          0
  -> 10.10.36.7:80                Tunnel  100    0          0

#!/bin/bash

TYPE=$1
NAME=$2
STATE=$3
FILE="/etc/keepalived/${NAME}"

if [ ! -f "${FILE}" ]; then
  touch "${FILE}"
fi

ORI_STATE=`cat ${FILE}`

if [ ${STATE} == ${ORI_STATE} ];then
   exit 0
else
    case $STATE in
            "MASTER") /bin/python /etc/keepalived/sendmail.py ${STATE} ${TYPE} ${NAME}
                      echo "${STATE}" > "${FILE}"
                      exit 0
                      ;;
            "BACKUP") /bin/python /etc/keepalived/sendmail.py ${STATE} ${TYPE} ${NAME}
                      echo "${STATE}" > "${FILE}"
                      exit 0
                      ;;
            "FAULT")  /bin/python /etc/keepalived/sendmail.py ${STATE} ${TYPE} ${NAME}
                      echo "${STATE}" > "${FILE}"
                      exit 0
                      ;;
            *)        echo "unknown state"
                      exit 1
                      ;;
    esac
fi