天天看點
傳回

NFS （NAS 網絡存儲）DRBD高可用NFS （NAS 網絡存儲）DRBD高可用

NFS （NAS 網絡存儲）DRBD高可用

安裝準備

  • 伺服器資訊
IP位址 角色、主機名
192.168.1.97 nfs-master.host.com
192.168.1.98 nfs-backup.host.com
192.168.1.10 keepalived VIP
  • 系統資訊 
Static hostname: nfs-master.host.com
         Icon name: computer-vm
           Chassis: vm
        Machine ID: d0a403dc7abd4a78b64dc5b22e4db7b4
           Boot ID: 164d9d5c045f4c2f9496aa40326b38a2
    Virtualization: vmware
  Operating System: CentOS Linux 7 (Core)
       CPE OS Name: cpe:/o:centos:centos:7
            Kernel: Linux 3.10.0-1127.el7.x86_64
      Architecture: x86-64
  • 共享目錄配置 
Master 與 Bakcup 使用相同路徑 /private_data
  • 系統初始化 
cat >> /etc/sysctl.d/nfs.conf << EOF
vm.overcommit_memory = 1
net.ipv4.ip_local_port_range = 1024 65536
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_abort_on_overflow = 0
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 262144
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 262144
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.ipv4.netfilter.ip_conntrack_max = 2097152
net.nf_conntrack_max = 655360
net.netfilter.nf_conntrack_tcp_timeout_established = 1200
EOF
/sbin/sysctl -p /etc/sysctl.d/nfs.conf
# 執行如果出現報錯（nf_conntrack_max no such file or directory）執行（modprobe ip_conntrack）重新/sbin/sysctl -p /etc/sysctl.d/nfs.conf即可

安裝配置DRBD與 NFS

以下安裝在 2 台伺服器執行

  • 安裝 nfs 
yum -y install nfs-utils rpcbind
  • 配置安裝DRBD 
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
yum install drbd84 kmod-drbd84 -y
  • 加載子產品 
    modprobe drbd
#檢視子產品是否加載上了
lsmod |grep drbd
  • 檢視修改配置檔案 
    # You can find an example in  /usr/share/doc/drbd.../drbd.conf.example

#include "drbd.d/global_common.conf";
include "drbd.d/*.res";
  • 添加資源配置檔案 
    global {
        usage-count yes;
        #是否參與DRBD使用者統計，預設為yes
}

common {
        syncer { rate 30M; }
}
#設定主備節點同步的網絡速率最大值，預設機關是位元組，我們可以設定為兆
    
resource r0 {
        #r0為資源名，我們在初始化磁盤的時候就可以使用資源名來初始化。
        protocol C;
        #使用 C 協定。
        handlers {
            pri-on-incon-degr "echo o > /proc/sysrq-trigger ; halt -f ";  
            pri-lost-after-sb "echo o > /proc/sysrq-trigger ; halt ";
            local-io-error "echo o > /proc/sysrq-trigger ; halt -f";
            fence-peer "/usr/lib4/heartbeat/drbd-peer-outdater -t 5";
            pri-lost "echo pri-lst. Have a look at the log file.mail -s 'Drbd Alert' root";
            split-brain "/usr/lib/drbd/notify-split-brain.sh root";
            out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
        }

        net {
          cram-hmac-alg "sha1";
          shared-secret "NFS-HA";
          #drbd同步時使用的驗證方式和密碼資訊
        }

        disk {
          on-io-error detach;
          fencing resource-only;
          # 使用DOPD（drbd outdate-peer deamon）功能保證資料不同步的時候不進行切換。
        }

        startup {
          wfc-timeout 120;
          degr-wfc-timeout 120;
        }

        device /dev/drbd0;
        #這裡/dev/drbd0是使用者挂載時的裝置名字，由DRBD程序建立
        on nfs-master.host.com {
        #每個主機名的說明以on開頭，後面是hostname（必須在/etc/hosts可解析）
          disk /dev/sdb1;
          #使用這個磁盤作為drbd的磁盤/dev/drbd0。
          address 192.168.1.97:7788;
          #設定DRBD的監聽端口，用于與另一台主機通信
          meta-disk internal;
          #drbd的中繼資料存放方式
        } 

        on nfs-backup.host.com {
          disk /dev/sdb1;
          address 192.168.1.98:7788;
          meta-disk internal;
        }
}
  • 硬碟分區，不進行格式化(在 Master 執行即可) 
    fdisk /dev/sdb
n----->p------>1------->回車---->回車------>w
  • 建立DRBD裝置并激活r0資源 
    # 建立drbd 裝置 
mknod /dev/drbd0 b 147 0
#激活r0資源
drbdadm create-md r0
# 根據提示進行操作
WARN:
  You are using the 'drbd-peer-outdater' as fence-peer program.
  If you use that mechanism the dopd heartbeat plugin program needs
  to be able to call drbdsetup and drbdmeta with root privileges.

  You need to fix this with these commands:
  chgrp haclient /lib/drbd/drbdsetup-84
  chmod o-x /lib/drbd/drbdsetup-84
  chmod u+s /lib/drbd/drbdsetup-84

  chgrp haclient /usr/sbin/drbdmeta
  chmod o-x /usr/sbin/drbdmeta
  chmod u+s /usr/sbin/drbdmeta

initializing activity log
initializing bitmap (640 KB) to all zero
Writing meta data...
New drbd meta data block successfully created.

# useradd -M -s /sbin/nologin haclient   #建立程式使用者
# chgrp haclient /lib/drbd/drbdsetup-84
# chmod o-x /lib/drbd/drbdsetup-84
# chmod u+s /lib/drbd/drbdsetup-84
    
# chgrp haclient /usr/sbin/drbdmeta
#  chmod o-x /usr/sbin/drbdmeta
#  chmod u+s /usr/sbin/drbdmeta
    # 再次激活裝置
# drbdadm create-md r0
    • 啟動DRBD服務 
      systemctl start drbd && systemctl enable drbd
    • 檢視 drbd 狀态 
      cat /proc/drbd
version: 8.4.11-1 (api:1/proto:86-101)
GIT-hash: 66145a308421e9c124ec391a7848ac20203bb03c build by [email protected], 2020-04-05 02:58:18
 0: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r-----
    ns:1276152 nr:63137180 dw:64411784 dr:1877 al:361 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:459931480
        [=>..................] sync'ed: 12.3% (449148/511980)M
        finish: 3:04:38 speed: 41,512 (40,516) want: 41,040 K/sec
    • 配置主伺服器，設定DRBD主(nfs-master) 
      drbdsetup /dev/drbd0 primary --force  #在主伺服器執行，設定為主
# cat /proc/drbd
version: 8.4.11-1 (api:1/proto:86-101)
GIT-hash: 66145a308421e9c124ec391a7848ac20203bb03c build by [email protected], 2020-04-05 02:58:18
 0: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r-----
    ns:86032412 nr:1276188 dw:8334276 dr:78988028 al:1811 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:437037960
        [==>.................] sync'ed: 16.7% (426792/511980)M
        finish: 3:03:00 speed: 39,780 (39,904) K/sec

Primary/Secondary  表示目前為主，另一台為備
    • 在主伺服器上，挂載磁盤，(nfs-master) 
      ## Master 與 Bakcup 使用相同路徑 /private_data
[[email protected] ~]# mkdir /private_data
[[email protected] ~]# mkfs.ext4 /dev/drbd0  #格式化磁盤
[[email protected] ~]# mount /dev/drbd0 /private_data  #挂載磁盤
    • 手動切換進行測試 
      #主伺服器
[[email protected] ~]# touch /private_data/er
[[email protected] ~]# ls /private_data
er  lost+found
[[email protected] ~]# umount /private_data   #取消挂載
[[email protected] ~]#  drbdsetup /dev/drbd0 secondary  #将主伺服器，DRBD設定為備
#備伺服器
[[email protected] ~]# mkdir /private_data   #建立共享目錄
[[email protected] ~]#  drbdsetup /dev/drbd0 primary   #将備伺服器中DRBD設定為主
[[email protected] ~]# mount /dev/drbd0 /private_data  #挂載硬碟
[[email protected] ~]# ls /private_data  #檢視
er  lost+found  #檔案實作了備份

配置 NFS高可用

  • 配置 keepalived 
# 安裝 keepalived
yum install keepalived -y
# 配置（Master）
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.back && cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
  notification_email {
    [email protected]
    }
     
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DRBD_HA_MASTER
}
     
vrrp_script chk_nfs {
        script "/etc/keepalived/check_nfs.sh"
        interval 5
    }
    vrrp_instance VI_1 {
        state MASTER
        interface eth0
        virtual_router_id 101
        priority 100
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        track_script {
            chk_nfs
        }
    notify_stop /etc/keepalived/notify_stop.sh       
    notify_master /etc/keepalived/notify_master.sh   
    virtual_ipaddress {
        192.168.1.10/24
    }
}
# 配置（Slave）
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.back && cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
  notification_email {
    [email protected]
    }
     
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DRBD_HA_BACKUP
}
     
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 101
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    # 當進入Master狀态時會呼叫notify_master

    # 當進入Backup狀态時會呼叫notify_backup

    # 當發現異常情況時進入Fault狀态呼叫notify_fault

    # 當Keepalived程式終止時則呼叫notify_stop

    notify_master /etc/keepalived/notify_master.sh            
  
    notify_backup /etc/keepalived/notify_backup.sh          
  
    virtual_ipaddress {
        192.168.1.10/24
    }
}
# 啟動 keepalived(Master、Backup)
systemctl start keepalived && systemctl enable keepalived
  • Keepalived 檢測腳本 
[[email protected] keepalived]# cat check_nfs.sh 
#!/usr/bin/env bash 

###檢查nfs可用性：程序和是否能夠挂載
/usr/bin/systemctl status nfs &>/dev/null
if [ $? -ne 0 ];then
    ###如果服務狀态不正常，先嘗試重新開機服務
    /usr/bin/systemctl restart nfs
    /usr/bin/systemctl status nfs &>/dev/null
    if [ $? -ne 0 ];then
        ###若重新開機nfs服務後，仍不正常
        ###解除安裝drbd裝置
        umount /dev/drbd0
        ###将drbd主降級為備
        drbdadm secondary r0
        #關閉keepalived
        /usr/bin/systemctl stop keepalived
    fi
fi
cat notify_master.sh
#!/usr/bin/env bash
cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_mount=$(whereis mount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")

time=$(date "+%F  %H:%M:%S")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time    ------notify_master------\n" >> /etc/keepalived/logs/notify_master.log
    ${cmd_drbdadm} primary r0 &>> /etc/keepalived/logs/notify_master.log
    ${cmd_mount} /dev/drbd0 /private_data &>> /etc/keepalived/logs/notify_master.log
    ${cmd_service} restart nfs &>> /etc/keepalived/logs/notify_master.log
    echo -e "\n" >> /etc/keepalived/logs/notify_master.log
fi

cat notify_stop.sh
#!/usr/bin/env bash

cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_umount=$(whereis umount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")

time=$(date "+%F  %H:%M:%S")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time  ------notify_stop------\n" >> /etc/keepalived/logs/notify_stop.log
    ${cmd_service} stop nfs &>> /etc/keepalived/logs/notify_stop.log
    ${cmd_umount} /private_data &>> /etc/keepalived/logs/notify_stop.log
    ${cmd_drbdadm} secondary r0 &>> /etc/keepalived/logs/notify_stop.log
    echo -e "\n" >> /etc/keepalived/logs/notify_stop.log
fi
### backup
#!/usr/bin/env bash

cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_mount=$(whereis mount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")
time=$(date "+%F  %H:%M:%S")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time    ------notify_master------\n" >> /etc/keepalived/logs/notify_master.log
    ${cmd_drbdadm} primary r0 &>> /etc/keepalived/logs/notify_master.log
    ${cmd_mount} /dev/drbd0 /private_data &>> /etc/keepalived/logs/notify_master.log
    ${cmd_service} restart nfs &>> /etc/keepalived/logs/notify_master.log
    echo -e "\n" >> /etc/keepalived/logs/notify_master.log
fi

#!/usr/bin/env bash

cmd_drbdadm=$(whereis drbdadm | awk '{print $2}' | tr -d "\n")
cmd_umount=$(whereis umount | awk '{print $2}' | tr -d "\n")
cmd_service=$(whereis systemctl | awk '{print $2}' | tr -d "\n")
if [[ -n "${cmd_drbdadm}" ]]; then
    echo -e "$time    ------notify_backup------\n" >> /etc/keepalived/logs/notify_backup.log
    ${cmd_service} stop nfs &>> /etc/keepalived/logs/notify_backup.log
    ${cmd_umount} /dev/drbd0 &>> /etc/keepalived/logs/notify_backup.log
    ${cmd_drbdadm} secondary r0 &>> /etc/keepalived/logs/notify_backup.log
    echo -e "\n" >> /etc/keepalived/logs/notify_backup.log
fi
Linux 運維 nas Linux DRBD
上一篇: 【Servlet】利用JSTL表達式把Servlet從資料庫讀出的資料推向前台與javax.el.PropertyNotFoundException的錯誤
下一篇: 昨天在大街上，突然聽到有人喊我的名字+老師，我看着他，愣了好久，而我還帶着口罩，他沒有帶，他認出了我，我怎麼都想不起來他

繼續閱讀