keeplive+haproxy+nginx

Keepalivedt做高可用，其高可用，都是站在伺服器腳本去說的高可用，而不是服務的角度,也就是說，如果伺服器DOWN機或者網絡出現故障，高可用是可以實作自動切換的。如果運作的服務，比如Nginx挂掉,這些高可用軟體是意識不到的，需要自己寫腳本去實作服務的切換

master        nginx+keeplive+haproxy  192.168.1.108

master/slave    nginx+keeplive+haproxy  192.168.1.103

vip          192.168.1.200

實驗環境

centos_6.8(x64)

實驗軟體

pcre-8.10.tar.gz

nginx-1.1.1.tar.gz

keepalived-1.2.1.tar.gz

haproxy-1.4.8.tar.gz

軟體安裝

yum install -y gcc gcc-c++* make* cmake* zlib* openssl* popt* nmap* ntp* lrzsz* autoconf* automake* libxml*  ncurses* libmcrypt* ipvsadm* libnl-devel* kernel-devel

/etc/rc.d/init.d/ntpd start

ntpdate 伺服器IP

clock -w        

chkconfig --level 35 ntpd on

tar zxvf pcre-8.10.tar.gz

cd pcre-8.10

./configure

make && make install

tar zxvf nginx-1.1.1.tar.gz

cd nginx-1.1.1

./configure --prefix=/usr/local/nginx --with-http_stub_status_module

/usr/local/nginx/sbin/nginx -t  檢查nginx配置

nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok

nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successfu

如果看到這行提示，證明nginx配置沒有錯誤

/usr/local/nginx/sbin/nginx -s reload

nginx: [error] invalid PID number "" in "/usr/local/nginx/logs/nginx.pid"

/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf

/usr/local/nginx/sbin/nginx -s reload        nginx平滑重新開機

cp /usr/local/nginx/sbin/nginx /etc/init.d/

chmod +x /etc/init.d/nginx 

/etc/rc.d/init.d/nginx                  啟動nginx

/usr/local/nginx/sbin/nginx &             也可以使用這種辦法啟動nginx

cp /usr/local/nginx/html/index.html /root/    

echo > /usr/local/nginx/html/index.html

echo web1 > /usr/local/nginx/html/index.html  master端192.168.1.108

tail /usr/local/nginx/html/index.html 

web1

echo web2 > /usr/local/nginx/html/index.html  slave端192.168.1.103操作

web2

tar zxvf keepalived-1.2.1.tar.gz

cd keepalived-1.2.1

ll /usr/src/kernels/

total 4

drwxr-xr-x 22 root root 4096 May  8 18:01 2.6.32-696.1.1.el6.x86_64

./configure --prefix=/usr/local/keeplived 

--with-kernel-dir=/usr/src/kernels/2.6.32-696.1.1.el6.x86_64

Keepalived configuration

------------------------

Keepalived version       : 1.2.1

Compiler                 : gcc

Compiler flags           : -g -O2

Extra Lib                : -lpopt -lssl -lcrypto 

Use IPVS Framework       : Yes

IPVS sync daemon support : Yes

Use VRRP Framework       : Yes  3個必須是yes

Use Debug flags          : No

make  &&  make install

cp /usr/local/keeplived/etc/rc.d/init.d/keepalived   /etc/rc.d/init.d/

chmod +x /etc/rc.d/init.d/keepalived

cp /usr/local/keeplived/etc/sysconfig/keepalived /etc/sysconfig/

cp /usr/local/keeplived/etc/keepalived/keepalived.conf /etc/keepalived/

cp /usr/local/keeplived/sbin/keepalived /usr/sbin/

chmod +x /usr/sbin/keepalived

echo "/usr/local/nginx/sbin/nginx" >>  /etc/rc.local

echo "/etc/init.d/keepalived start" >>  /etc/rc.local  開機啟動服務

/etc/init.d/keepalived start

Starting keepalived: [  OK  ]

tar zxvf haproxy-1.4.8.tar.gz 

cd haproxy-1.4.8

uname -a

Linux centos6 2.6.32-642.el6.x86_64 #1 SMP Tue May 10 17:27:01 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux

make TARGET=linux26 PREFIX=/usr/local/haproxy

make install PREFIX=/usr/local/haproxy

useradd -s /sbin/nologin haproxy

chown -R haproxy.haproxy /usr/local/haproxy

ll /usr/local/haproxy/

total 12

drwxr-xr-x 3 haproxy haproxy 4096 May  8 16:36 doc

drwxr-xr-x 2 haproxy haproxy 4096 May  8 16:36 sbin

drwxr-xr-x 3 haproxy haproxy 4096 May  8 16:36 share   預設沒有.cfg 配置檔案

touch /usr/local/haproxy/haproxy.cfg

vim  /usr/local/haproxy/haproxy.cfg

global

log 127.0.0.1 local0

maxconn 5120

chroot /usr/local/haproxy

user haproxy

group haproxy

daemon

quiet

nbproc  1

pidfile /usr/local/haproxy/haproxy.pid

debug

defaults

log 127.0.0.1 local3

mode http

option httplog

option httpclose

option  dontlognull

#option  forwardfor

option  redispatch

retries 2

maxconn 2000

balance source

contimeout      5000

clitimeout      50000

srvtimeout      50000

listen web_proxy :8080  web頁面配置 nginx預設端口80造成IP沖突 修改為8080

     server www1  nginx_master伺服器IP:80  weight 5 check inter 2000 rise 2 fall 5

     server www2  nginx_slave伺服器IP:80  weight 5 check inter 2000 rise 2 fall 5

listen  mysql

      bind 0.0.0.0:7306  7306為代理資料庫虛拟端口

      mode tcp

      server mysql 後端資料庫IP:3306

#transparent

stats uri / haproxy-stats

stats realm Haproxy \ statisti

/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg & 啟動服務

pkill haproxy  殺死程序

touch  /etc/init.d/haproxy

vim   /etc/init.d/haproxy

-----------------------------------------------------------------------

#!/bin/bash

#

# haproxy

# chkconfig: 35 85 15

# description: HAProxy is a free, very fast and reliable solution \

# offering high availability, load balancing, and \

# proxying for TCP and HTTP-based applications

# processname: haproxy

# config: /usr/local/haproxy/haproxy.cfg  haproxy配置檔案目錄

# pidfile: /usr/local/haproxy/haproxy.pid

# Source function library.

. /etc/rc.d/init.d/functions

# Source networking configuration.

. /etc/sysconfig/network

# Check that networking is up.

[ "$NETWORKING" = "no" ] && exit 0

config="/usr/local/haproxy/haproxy.cfg"  haproxy 配置檔案目錄

exec="/usr/local/haproxy/sbin/haproxy"

prog=$(basename $exec)

[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog

lockfile=/var/lock/subsys/haproxy

check() {

    $exec -c -V -f $config

}

start() {

    $exec -c -q -f $config

    if [ $? -ne 0 ]; then

        echo "Errors in configuration file, check with $prog check."

        return 1

    fi

    echo -n $"Starting $prog: "

    # start it up here, usually something like "daemon $exec"

    daemon $exec -D -f $config -p /var/run/$prog.pid

    retval=$?

    echo

    [ $retval -eq 0 ] && touch $lockfile

    return $retval

stop() {

    echo -n $"Stopping $prog: "

    # stop it here, often "killproc $prog"

    killproc $prog 

    [ $retval -eq 0 ] && rm -f $lockfile

restart() {

    stop

    start

reload() {

    echo -n $"Reloading $prog: "

    $exec -D -f $config -p /var/run/$prog.pid -sf $(cat /var/run/$prog.pid)

force_reload() {

    restart

fdr_status() {

    status $prog

case "$1" in

    start|stop|restart|reload)

        $1

        ;;

    force-reload)

        force_reload

    checkconfig)

        check

    status)

        fdr_status

    condrestart|try-restart)

      [ ! -f $lockfile ] || restart

    ;;

    *)

        echo $"Usage: $0 {start|stop|status|checkconfig|restart|try-restart|reload|force-reload}"

        exit 2

esac

-----------------------------------------------------------------------------

chmod +x /etc/init.d/haproxy  使用腳本方式重新開機服務

ps -aux | grep haporxy

Warning: bad syntax, perhaps a bogus '-'? See /usr/share/doc/procps-3.2.8/FAQ

haproxy   9165  0.1  0.1  14072  1856 pts/0    S    17:01   0:00 /usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg

root      9167  0.0  0.0 103316   836 pts/0    S+   17:05   0:00 grep haproxy

netstat -tuplna | grep nginx

tcp        0      0 0.0.0.0:80                  0.0.0.0:*                   LISTEN      11343/nginx

ps -aux | grep nginx

root     38686  0.0  0.1  20160  1180 ?        Ss   10:48   0:00 nginx: master process /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf

nobody   38691  0.0  0.1  20544  1284 ?        S    10:49   0:00 nginx: worker process                                          

root     38711  0.0  0.0 103260   836 pts/1    S+   10:51   0:00 grep nginx

以上所有操作 master端點 和slave端相同

<a href="http://blog.51cto.com/haproxy-stats">http://伺服器IP:8080/haproxy-stats</a>

<a href="https://s3.51cto.com/wyfs02/M01/94/F8/wKiom1kQNcnQ9CdRAAEp0omKgAI046.jpg-wh_500x0-wm_3-wmp_4-s_904051372.jpg" target="_blank"></a>

<a href="https://s2.51cto.com/wyfs02/M01/94/0E/wKioL1kLRUCxzYQ6AAB3aZ5FiwA285.jpg-wh_500x0-wm_3-wmp_4-s_1570688293.jpg" target="_blank"></a>

vim /etc/keepalived/keepalived.conf    master端 192.168.1.108配置

! Configuration File for keepalived

global_defs {

   notification_email {

     [email protected]  通告郵箱

   }

   notification_email_from  [email protected]

   smtp_server smtp.126.com

   smtp_connect_timeout 30

   router_id master

vrrp_script chk_http_port {

    script "/root/chk_nginx.sh"

    interval 2

    weight 2

track_script {

    chk_http_port

vrrp_instance VI_1 {

    state MASTER

    interface eth0

    virtual_router_id 101

    priority 100

    advert_int 1

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    virtual_ipaddress {

        192.168.1.200  虛拟IP

vim /etc/keepalived/keepalived.conf  slave端 192.168.1.103 配置

     [email protected]  通告郵箱

   notification_email_from  [email protected]

   router_id backup

    state MASTER/BACKUP    master為主主模式 backup為主從模式

    virtual_router_id 101   主主 主從 模式route_id相同

    priority 100/99       主主 100/100優先級相同 主從100/99從優先級比主小

        192.168.1.200     虛拟IP

/etc/rc.d/init.d/keepalived restart

touch chk_nginx.sh

chmod +x chk_nginx.sh

vim chk_nginx.sh

#!/bin/sh

# check nginx server status

NGINX=/usr/local/nginx/sbin/nginx

PORT=80

nmap localhost -p $PORT | grep "$PORT/tcp open"

#echo $?

if [ $? -ne 0 ];then

    $NGINX -s stop

    $NGINX

    sleep 3

    nmap localhost -p $PORT | grep "$PORT/tcp open"

    [ $? -ne 0 ] &amp;&amp; /etc/init.d/keepalived stop

fi

sh chk_nginx.sh 

80/tcp open  http

vim /etc/crontab

*/5  *  *  *  * sh /root/root/chk_nginx.sh

service crond restart

vim /etc/sysctl.conf

net.ipv4.ip_forward = 1 修改為1 打開路由轉發功能

sysctl -p

net.ipv4.ip_forward = 1

[root@centos6 ~]# ip a                master端192.168.1.108配置

 eth0: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc pfifo_fast state UP qlen 1000

    link/ether 00:0c:29:45:17:69 brd ff:ff:ff:ff:ff:ff

    inet 192.168.1.108/24 brd 192.168.1.255 scope global eth0

    inet 192.168.1.200/32 scope global eth0      虛拟ip啟動成功

    inet6 fe80::20c:29ff:fe45:1769/64 scope link 

    valid_lft forever preferred_lft forever

[root@centos6-1 ~]# ip a               slave端192.168.1.103配置

  eth0: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc pfifo_fast state UP qlen 1000

    link/ether 00:0c:29:3d:5b:1b brd ff:ff:ff:ff:ff:ff

    inet 192.168.1.103/24 brd 192.168.1.255 scope global eth0

    inet 192.168.1.200/24 scope global secondary eth0  虛拟ip啟動成功

    inet6 fe80::20c:29ff:fe3d:5b1b/64 scope link 

       valid_lft forever preferred_lft forever

本文轉自 mailfile 51CTO部落格，原文連結：http://blog.51cto.com/mailfile/1329362，如需轉載請自行聯系原作者