天天看点
返回

NFS （NAS 网络存储）DRBD高可用NFS （NAS 网络存储）DRBD高可用

NFS （NAS 网络存储）DRBD高可用

安装准备

  • 服务器信息
IP地址 角色、主机名
192.168.1.97 nfs-master.host.com
192.168.1.98 nfs-backup.host.com
192.168.1.10 keepalived VIP
  • 系统信息 
Static hostname: nfs-master.host.com
         Icon name: computer-vm
           Chassis: vm
        Machine ID: d0a403dc7abd4a78b64dc5b22e4db7b4
           Boot ID: 164d9d5c045f4c2f9496aa40326b38a2
    Virtualization: vmware
  Operating System: CentOS Linux 7 (Core)
       CPE OS Name: cpe:/o:centos:centos:7
            Kernel: Linux 3.10.0-1127.el7.x86_64
      Architecture: x86-64
  • 共享目录配置 
Master 与 Bakcup 使用相同路径 /private_data
  • 系统初始化 
cat >> /etc/sysctl.d/nfs.conf << EOF
vm.overcommit_memory = 1
net.ipv4.ip_local_port_range = 1024 65536
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_abort_on_overflow = 0
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 262144
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 262144
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.ipv4.netfilter.ip_conntrack_max = 2097152
net.nf_conntrack_max = 655360
net.netfilter.nf_conntrack_tcp_timeout_established = 1200
EOF
/sbin/sysctl -p /etc/sysctl.d/nfs.conf
# 执行如果出现报错（nf_conntrack_max no such file or directory）执行（modprobe ip_conntrack）重新/sbin/sysctl -p /etc/sysctl.d/nfs.conf即可

安装配置DRBD与 NFS

以下安装在 2 台服务器执行

  • 安装 nfs 
yum -y install nfs-utils rpcbind
  • 配置安装DRBD 
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
yum install drbd84 kmod-drbd84 -y
  • 加载模块 
    modprobe drbd
#查看模块是否加载上了
lsmod |grep drbd
  • 查看修改配置文件 
    # You can find an example in  /usr/share/doc/drbd.../drbd.conf.example

#include "drbd.d/global_common.conf";
include "drbd.d/*.res";
  • 添加资源配置文件 
    global {
        usage-count yes;
        #是否参与DRBD使用者统计，默认为yes
}

common {
        syncer { rate 30M; }
}
#设置主备节点同步的网络速率最大值，默认单位是字节，我们可以设定为兆
    
resource r0 {
        #r0为资源名，我们在初始化磁盘的时候就可以使用资源名来初始化。
        protocol C;
        #使用 C 协议。
        handlers {
            pri-on-incon-degr "echo o > /proc/sysrq-trigger ; halt -f ";  
            pri-lost-after-sb "echo o > /proc/sysrq-trigger ; halt ";
            local-io-error "echo o > /proc/sysrq-trigger ; halt -f";
            fence-peer "/usr/lib4/heartbeat/drbd-peer-outdater -t 5";
            pri-lost "echo pri-lst. Have a look at the log file.mail -s 'Drbd Alert' root";
            split-brain "/usr/lib/drbd/notify-split-brain.sh root";
            out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
        }

        net {
          cram-hmac-alg "sha1";
          shared-secret "NFS-HA";
          #drbd同步时使用的验证方式和密码信息
        }

        disk {
          on-io-error detach;
          fencing resource-only;
          # 使用DOPD（drbd outdate-peer deamon）功能保证数据不同步的时候不进行切换。
        }

        startup {
          wfc-timeout 120;
          degr-wfc-timeout 120;
        }

        device /dev/drbd0;
        #这里/dev/drbd0是用户挂载时的设备名字，由DRBD进程创建
        on nfs-master.host.com {
        #每个主机名的说明以on开头，后面是hostname（必须在/etc/hosts可解析）
          disk /dev/sdb1;
          #使用这个磁盘作为drbd的磁盘/dev/drbd0。
          address 192.168.1.97:7788;
          #设置DRBD的监听端口，用于与另一台主机通信
          meta-disk internal;
          #drbd的元数据存放方式
        } 

        on nfs-backup.host.com {
          disk /dev/sdb1;
          address 192.168.1.98:7788;
          meta-disk internal;
        }
}
  • 硬盘分区，不进行格式化(在 Master 执行即可) 
    fdisk /dev/sdb
n----->p------>1------->回车---->回车------>w
  • 创建DRBD设备并激活r0资源 
    # 创建drbd 设备 
mknod /dev/drbd0 b 147 0
#激活r0资源
drbdadm create-md r0
# 根据提示进行操作
WARN:
  You are using the 'drbd-peer-outdater' as fence-peer program.
  If you use that mechanism the dopd heartbeat plugin program needs
  to be able to call drbdsetup and drbdmeta with root privileges.

  You need to fix this with these commands:
  chgrp haclient /lib/drbd/drbdsetup-84
  chmod o-x /lib/drbd/drbdsetup-84
  chmod u+s /lib/drbd/drbdsetup-84

  chgrp haclient /usr/sbin/drbdmeta
  chmod o-x /usr/sbin/drbdmeta
  chmod u+s /usr/sbin/drbdmeta

initializing activity log
initializing bitmap (640 KB) to all zero
Writing meta data...
New drbd meta data block successfully created.

# useradd -M -s /sbin/nologin haclient   #创建程序用户
# chgrp haclient /lib/drbd/drbdsetup-84
# chmod o-x /lib/drbd/drbdsetup-84
# chmod u+s /lib/drbd/drbdsetup-84
    
# chgrp haclient /usr/sbin/drbdmeta
#  chmod o-x /usr/sbin/drbdmeta
#  chmod u+s /usr/sbin/drbdmeta
    # 再次激活设备
# drbdadm create-md r0
    • 启动DRBD服务 
      systemctl start drbd && systemctl enable drbd
    • 查看 drbd 状态 
      cat /proc/drbd
version: 8.4.11-1 (api:1/proto:86-101)
GIT-hash: 66145a308421e9c124ec391a7848ac20203bb03c build by [email protected], 2020-04-05 02:58:18
 0: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r-----
    ns:1276152 nr:63137180 dw:64411784 dr:1877 al:361 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:459931480
        [=>..................] sync'ed: 12.3% (449148/511980)M
        finish: 3:04:38 speed: 41,512 (40,516) want: 41,040 K/sec
    • 配置主服务器，设置DRBD主(nfs-master) 
      drbdsetup /dev/drbd0 primary --force  #在主服务器执行，设置为主
# cat /proc/drbd
version: 8.4.11-1 (api:1/proto:86-101)
GIT-hash: 66145a308421e9c124ec391a7848ac20203bb03c build by [email protected], 2020-04-05 02:58:18
 0: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r-----
    ns:86032412 nr:1276188 dw:8334276 dr:78988028 al:1811 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:437037960
        [==>.................] sync'ed: 16.7% (426792/511980)M
        finish: 3:03:00 speed: 39,780 (39,904) K/sec

Primary/Secondary  表示当前为主，另一台为备
    • 在主服务器上，挂载磁盘，(nfs-master) 
      ## Master 与 Bakcup 使用相同路径 /private_data
[[email protected] ~]# mkdir /private_data
[[email protected] ~]# mkfs.ext4 /dev/drbd0  #格式化磁盘
[[email protected] ~]# mount /dev/drbd0 /private_data  #挂载磁盘
    • 手动切换进行测试 
      #主服务器
[[email protected] ~]# touch /private_data/er
[[email protected] ~]# ls /private_data
er  lost+found
[[email protected] ~]# umount /private_data   #取消挂载
[[email protected] ~]#  drbdsetup /dev/drbd0 secondary  #将主服务器，DRBD设置为备
#备服务器
[[email protected] ~]# mkdir /private_data   #创建共享目录
[[email protected] ~]#  drbdsetup /dev/drbd0 primary   #将备服务器中DRBD设置为主
[[email protected] ~]# mount /dev/drbd0 /private_data  #挂载硬盘
[[email protected] ~]# ls /private_data  #查看
er  lost+found  #文件实现了备份

配置 NFS高可用

  • 配置 keepalived 
# 安装 keepalived
yum install keepalived -y
# 配置（Master）
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.back && cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
  notification_email {
    [email protected]
    }
     
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DRBD_HA_MASTER
}
     
vrrp_script chk_nfs {
        script "/etc/keepalived/check_nfs.sh"
        interval 5
    }
    vrrp_instance VI_1 {
        state MASTER
        interface eth0
        virtual_router_id 101
        priority 100
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        track_script {
            chk_nfs
        }
    notify_stop /etc/keepalived/notify_stop.sh       
    notify_master /etc/keepalived/notify_master.sh   
    virtual_ipaddress {
        192.168.1.10/24
    }
}
# 配置（Slave）
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.back && cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
  notification_email {
    [email protected]
    }
     
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DRBD_HA_BACKUP
}
     
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 101
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    # 当进入Master状态时会呼叫notify_master

    # 当进入Backup状态时会呼叫notify_backup

    # 当发现异常情况时进入Fault状态呼叫notify_fault

    # 当Keepalived程序终止时则呼叫notify_stop

    notify_master /etc/keepalived/notify_master.sh            
  
    notify_backup /etc/keepalived/notify_backup.sh          
  
    virtual_ipaddress {
        192.168.1.10/24
    }
}
# 启动 keepalived(Master、Backup)
systemctl start keepalived && systemctl enable keepalived
  • Keepalived 检测脚本 
[[email protected] keepalived]# cat check_nfs.sh 
#!/usr/bin/env bash 

###检查nfs可用性：进程和是否能够挂载
/usr/bin/systemctl status nfs &>/dev/null
if [ $? -ne 0 ];then
    ###如果服务状态不正常，先尝试重启服务
    /usr/bin/systemctl restart nfs
    /usr/bin/systemctl status nfs &>/dev/null
    if [ $? -ne 0 ];then
        ###若重启nfs服务后，仍不正常
        ###卸载drbd设备
        umount /dev/drbd0
        ###将drbd主降级为备
        drbdadm secondary r0
        #关闭keepalived
        /usr/bin/systemctl stop keepalived
    fi
fi
cat notify_master.sh
#!/usr/bin/env bash
cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_mount=$(whereis mount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")

time=$(date "+%F  %H:%M:%S")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time    ------notify_master------\n" >> /etc/keepalived/logs/notify_master.log
    ${cmd_drbdadm} primary r0 &>> /etc/keepalived/logs/notify_master.log
    ${cmd_mount} /dev/drbd0 /private_data &>> /etc/keepalived/logs/notify_master.log
    ${cmd_service} restart nfs &>> /etc/keepalived/logs/notify_master.log
    echo -e "\n" >> /etc/keepalived/logs/notify_master.log
fi

cat notify_stop.sh
#!/usr/bin/env bash

cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_umount=$(whereis umount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")

time=$(date "+%F  %H:%M:%S")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time  ------notify_stop------\n" >> /etc/keepalived/logs/notify_stop.log
    ${cmd_service} stop nfs &>> /etc/keepalived/logs/notify_stop.log
    ${cmd_umount} /private_data &>> /etc/keepalived/logs/notify_stop.log
    ${cmd_drbdadm} secondary r0 &>> /etc/keepalived/logs/notify_stop.log
    echo -e "\n" >> /etc/keepalived/logs/notify_stop.log
fi
### backup
#!/usr/bin/env bash

cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_mount=$(whereis mount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")
time=$(date "+%F  %H:%M:%S")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time    ------notify_master------\n" >> /etc/keepalived/logs/notify_master.log
    ${cmd_drbdadm} primary r0 &>> /etc/keepalived/logs/notify_master.log
    ${cmd_mount} /dev/drbd0 /private_data &>> /etc/keepalived/logs/notify_master.log
    ${cmd_service} restart nfs &>> /etc/keepalived/logs/notify_master.log
    echo -e "\n" >> /etc/keepalived/logs/notify_master.log
fi

#!/usr/bin/env bash

cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_umount=$(whereis umount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time    ------notify_backup------\n" >> /etc/keepalived/logs/notify_backup.log
    ${cmd_service} stop nfs &>> /etc/keepalived/logs/notify_backup.log
    ${cmd_umount} /dev/drbd0 &>> /etc/keepalived/logs/notify_backup.log
    ${cmd_drbdadm} secondary r0 &>> /etc/keepalived/logs/notify_backup.log
    echo -e "\n" >> /etc/keepalived/logs/notify_backup.log
fi
Linux 运维 nas Linux DRBD
上一篇: 【Servlet】利用JSTL表达式把Servlet从数据库读出的数据推向前台与javax.el.PropertyNotFoundException的错误
下一篇: 昨天在大街上，突然听到有人喊我的名字+老师，我看着他，愣了好久，而我还带着口罩，他没有带，他认出了我，我怎么都想不起来他

继续阅读