文章目錄
- 一、前置準備
- 啟動redis服務
- 啟動sentinel服務
- 檢視sentiel啟動日志
- 二、實戰故障轉移
- 2.1. 主節點停止
- 2.2. Sentienl監控日志
- 2.3. 節點資訊監控
- 2.4. 故障轉移後變化
- 2.5. 重新啟動
- 2.6. sentine日志
- 2.7. 節點資訊監控
- 2.8. 配置檔案變化
- 2.9. 讀寫驗證
一、前置準備
啟動redis服務
# 檢視redis程序
ps -ef |grep redis
# 啟動redis服務
/usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf
#檢視主從節點資訊
啟動sentinel服務
聲明:在啟動3個redis服務的前提下,再依次啟動3個sentinel服務
/usr/local/redis/bin/redis-sentinel /usr/local/redis/conf/sentiel.conf
檢視sentiel啟動日志
tail
二、實戰故障轉移
2.1. 主節點停止
[root@bigdata01 redis-6.2.0]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replcation
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=xxx.xxx.92.101,port=6379,state=online,offset=3xxx,lag=1
slave1:ip=xxx.xxx.92.102,port=6379,state=online,offset=3xxx,lag=1
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3xxx
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:1778
127.0.0.1:6379> SHUTDOWN
not connected>
2.2. Sentienl監控日志
# +sdown master mymaster xxx.xxx.92.100 6379
啟動一個新選舉流程,記錄目前選舉的次數
# +new-epoch 1
投票選舉一個sentinel去做故障轉移這件事
# +vote-for-leader a20913bfcbeb3a5d8d58ce1bd372cc40455xxx10 1
客觀下線:目前環境内多個sentinel節點對我們的某一個伺服器标記為主觀下線,滿足quorum 仲裁的選項值之後,标記為客觀下線
# +odown master mymaster xxx.xxx.92.100 6379 #quorum 3/2
# Next failover delay: I will not start a failover before Sun Jul 4 22:39:56 2021
# +config-update-from sentinel a20913bfcbeb3a5d8d58ce1bd372cc40455xxx10 xxx.xxx.92.1026379
# +switch-master mymaster xxx.xxx.92.100 6379 xxx.xxx.92.101 6379
* +slave slave xxx.xxx.92.102:6379 xxx.xxx.92.102 6379 @ mymaster xxx.xxx.92.101 6379
* +slave slave xxx.xxx.92.100:6379 xxx.xxx.92.100 6379 @ mymaster xxx.xxx.92.101 6379
# +sdown slave xxx.xxx.92.100:6379 xxx.xxx.92.100 6379 @ mymaster xxx.xxx.92.101 6379
2.3. 節點資訊監控
檢視故障轉移後的節點資訊
01節點資訊
[root@bigdata02 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:3196
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3196
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:1806
# 故障轉移後 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:1
slave0:ip=xxx.xxx.92.102,port=6379,state=online,offset=380674,lag=1
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:380674
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:379284
127.0.0.1:6379>
02節點資訊
[root@bigdata03 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:3210
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3210
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:1778
# 故障轉移後 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:381546
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:381546
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:380114
127.0.0.1:6379>
2.4. 故障轉移後變化
01節點由slave節點變成了master節點,02仍然後slave節點
2.5. 重新啟動
故障的 100節點(原主節點)重新啟動
/usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf
2.6. sentine日志
* +convert-to-slave slave xxx.xxx.92.100:6379 xxx.xxx.92.100 6379 @ mymaster xxx.xxx.92.101 6379
2.7. 節點資訊監控
[root@bigdata01 redis-6.2.0]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:528510
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:528510
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:518136
repl_backlog_histlen:10375
127.0.0.1:6379>
01節點資訊
[root@bigdata02 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:3196
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3196
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:1806
# 故障轉移後 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:1
slave0:ip=xxx.xxx.92.102,port=6379,state=online,offset=380674,lag=1
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:380674
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:379284
#原主節點重新啟動後 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=xxx.xxx.92.102,port=6379,state=online,offset=537589,lag=1
slave1:ip=xxx.xxx.92.100,port=6379,state=online,offset=537732,lag=0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:537875
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:536485
127.0.0.1:6379>
02節點資訊
[root@bigdata03 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:3210
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3210
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:1778
# 故障轉移後 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:381546
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:381546
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:380114
#原主節點重新啟動後 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:556005
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:556005
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:554573
127.0.0.1:6379>
2.8. 配置檔案變化
原100主節點
# 放行通路IP的限制 多個用空格隔開
# 0.0.0.0 代表無限制
bind 0.0.0.0
# 背景啟動開啟
daemonize yes
# 日志存儲目錄以及日志檔案名
logfile "/usr/local/redis/log/redis.log"
# rdb資料檔案名
dbfilename "dump.rdb"
# aof模式開啟
appendonly no
# aof資料檔案名
appendfilename "appendonly.aof"
#rdb資料檔案和aof資料檔案存儲目錄
dir "/usr/local/redis/data"
# 設定密碼
requirepass 123456
# 從節點通路主節點的密碼(必須和requirepass 一緻)
masterauth "123456"
#從節點隻讀模式
replica-read-only yes
# Generated by CONFIG REWRITE
pidfile "/var/run/redis.pid"
save 3600 1
save 300 100
save 60 10000
user default on #8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92 ~* &* +@all
變化:最下面添加了主節點資訊
原01節點
# 放行通路IP的限制 多個用空格隔開
# 0.0.0.0 代表無限制
bind 0.0.0.0
# 背景啟動開啟
daemonize yes
# 日志存儲目錄以及日志檔案名
logfile "/usr/local/redis/log/redis.log"
# rdb資料檔案名
dbfilename "dump.rdb"
# aof模式開啟
appendonly no
# aof資料檔案名
appendfilename "appendonly.aof"
#rdb資料檔案和aof資料檔案存儲目錄
dir "/usr/local/redis/data"
# 設定密碼
requirepass 123456
# 從節點通路主節點的密碼(必須和requirepass 一緻)
masterauth "123456"
#從節點隻讀模式
replica-read-only yes
# 從節點從屬于那個主節點
# Generated by CONFIG REWRITE
pidfile "/var/run/redis.pid"
save 3600 1
save 300 100
save 60 10000
user default on #8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92 ~* &* +@all
# 放行通路IP的限制 多個用空格隔開
# 0.0.0.0 代表無限制
bind 0.0.0.0
# 背景啟動開啟
daemonize yes
# 日志存儲目錄以及日志檔案名
logfile "/usr/local/redis/log/redis.log"
# rdb資料檔案名
dbfilename "dump.rdb"
# aof模式開啟
appendonly no
# aof資料檔案名
appendfilename "appendonly.aof"
#rdb資料檔案和aof資料檔案存儲目錄
dir "/usr/local/redis/data"
# 設定密碼
requirepass 123456
# 從節點通路主節點的密碼(必須和requirepass 一緻)
masterauth "123456"
#從節點隻讀模式
replica-read-only yes
# 從節點從屬于那個主節點
replicaof xxx.xxx.92.101 6379
# Generated by CONFIG REWRITE
pidfile "/var/run/redis.pid"
save 3600 1
save 300 100
save 60 10000
user default on #8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92 ~* &* +@all
2.9. 讀寫驗證
# 登陸redis用戶端
#故障轉移後的100機器現在為slave節點,可讀不可寫
[root@bigdata01 redis-6.2.0]# /usr/local/redis/bin/redis-cli --user default --pass 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> set name lisi
(error) READONLY You can't write against a read only replica.
127.0.0.1:6379>
#故障轉移後的101機器現在為master節點,可以讀寫
[root@bigdata02 ~]# /usr/local/redis/bin/redis-cli --user default --pass 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> set name lisi
OK
127.0.0.1:6379> get name
"lisi"
127.0.0.1:6379>
#故障轉移後的102機器現在為slave節點,可讀不可寫
[root@bigdata03 ~]# /usr/local/redis/bin/redis-cli --user default --pass 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> set name wangwu
(error) READONLY You can't write against a read only replica.
127.0.0.1:6379> get name
"lisi"
127.0.0.1:6379>