ceph 添加新硬碟删除舊的OSD

接上一篇:

http://blog.chinaunix.net/uid-21142030-id-5194039.html

為每個OSD NODE添加一塊磁盤分區後再進行實驗.

[root@ceph_node1 osd]# df

Filesystem              1K-blocks    Used Available Use% Mounted on

/dev/mapper/centos-root   7022592 6789812    232780  97% /

devtmpfs                   933432       0    933432   0% /dev

tmpfs                      942208       0    942208   0% /dev/shm

tmpfs                      942208   32268    909940   4% /run

tmpfs                      942208       0    942208   0% /sys/fs/cgroup

/dev/vda1                  508588  139920    368668  28% /boot

/dev/vdb1                 8377344   32928   8344416   1% /osd

移除OSD,這裡隻做示範,實際生産環境中由于資料存放的原是以操作要慎重.

[root@ceph_monitor ~]# ceph osd out osd.3

marked out osd.3.

[root@ceph_monitor ~]# ceph osd out osd.4

marked out osd.4.

[root@ceph_monitor ~]# ceph osd down osd.4

marked down osd.4.

[root@ceph_monitor ~]# ceph osd down osd.3

marked down osd.3.

[root@ceph_monitor ~]# ceph osd rm osd.3  

osd.3 does not exist.

[root@ceph_monitor ~]# ceph osd rm osd.4  

removed osd.4

[root@ceph_node2 osd]# ceph -w

    cluster f35a65ad-1a6a-4e8d-8f7e-cb5f113c0a02

     health HEALTH_ERR

            64 pgs stale

            64 pgs stuck stale

            2 full osd(s)

     monmap e1: 1 mons at {ceph_monitor=10.0.2.33:6789/0}

            election epoch 1, quorum 0 ceph_monitor

     osdmap e22: 5 osds: 2 up, 2 in

            flags full

      pgmap v1644: 64 pgs, 1 pools, 0 bytes data, 0 objects

            13260 MB used, 455 MB / 13716 MB avail

                  64 stale+active+clean

2015-09-14 13:21:39.089705 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:21:49.988763 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:22:00.864299 mon.0 [INF] from='client.? 10.0.2.33:0/1014270' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:22:00.980026 mon.0 [INF] from='client.? 10.0.2.33:0/1014270' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["osd.3"]}]': finished

2015-09-14 13:22:01.043549 mon.0 [INF] osdmap e23: 5 osds: 2 up, 1 in full

2015-09-14 13:22:01.064547 mon.0 [INF] pgmap v1645: 64 pgs: 64 stale+active+clean; 0 bytes data, 6629 MB used, 228 MB / 6858 MB avail

2015-09-14 13:22:03.944095 mon.0 [INF] from='client.? 10.0.2.33:0/1014303' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:22:04.058603 mon.0 [INF] from='client.? 10.0.2.33:0/1014303' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["osd.4"]}]': finished

2015-09-14 13:22:04.136343 mon.0 [INF] osdmap e24: 5 osds: 2 up, 0 in full

2015-09-14 13:22:04.209347 mon.0 [INF] pgmap v1646: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:22:04.496508 mon.0 [INF] osdmap e25: 5 osds: 2 up, 0 in

2015-09-14 13:22:04.514888 mon.0 [INF] pgmap v1647: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:22:11.113375 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:22:19.412532 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:22:41.918295 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:22:51.015761 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:23:12.023552 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:23:15.296925 mon.0 [INF] pgmap v1648: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:23:19.301533 mon.0 [INF] pgmap v1649: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:23:22.419805 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:23:46.828958 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:23:54.523184 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:24:17.832867 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:24:27.627448 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:24:51.037841 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:24:57.930727 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:25:15.338544 mon.0 [INF] pgmap v1650: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:19.336484 mon.0 [INF] pgmap v1651: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:23.142214 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:25:27.034190 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:25:40.654246 mon.0 [INF] from='client.? 10.0.2.33:0/1014387' entity='client.admin' cmd=[{"prefix": "osd down", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:25:40.778345 mon.0 [INF] from='client.? 10.0.2.33:0/1014387' entity='client.admin' cmd='[{"prefix": "osd down", "ids": ["osd.4"]}]': finished

2015-09-14 13:25:40.801149 mon.0 [INF] osdmap e26: 5 osds: 1 up, 0 in

2015-09-14 13:25:40.820513 mon.0 [INF] pgmap v1652: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:42.980782 mon.0 [INF] osd.4 10.0.2.32:6800/4919 boot

2015-09-14 13:25:42.998345 mon.0 [INF] osdmap e27: 5 osds: 2 up, 0 in

2015-09-14 13:25:43.016972 mon.0 [INF] pgmap v1653: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:45.315515 mon.0 [INF] from='client.? 10.0.2.33:0/1014420' entity='client.admin' cmd=[{"prefix": "osd down", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:25:44.430084 osd.4 [WRN] map e26 wrongly marked me down

2015-09-14 13:25:45.428227 mon.0 [INF] from='client.? 10.0.2.33:0/1014420' entity='client.admin' cmd='[{"prefix": "osd down", "ids": ["osd.3"]}]': finished

2015-09-14 13:25:45.445942 mon.0 [INF] osdmap e28: 5 osds: 1 up, 0 in

2015-09-14 13:25:45.464796 mon.0 [INF] pgmap v1654: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:51.637408 mon.0 [INF] from='client.? 10.0.2.33:0/1014453' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:25:51.741785 mon.0 [INF] from='client.? 10.0.2.33:0/1014453' entity='client.admin' cmd='[{"prefix": "osd rm", "ids": ["osd.3"]}]': finished

2015-09-14 13:25:51.852460 mon.0 [INF] osdmap e29: 4 osds: 1 up, 0 in

2015-09-14 13:25:51.875071 mon.0 [INF] pgmap v1655: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:54.707323 mon.0 [INF] from='client.? 10.0.2.33:0/1014486' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:25:53.957396 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:25:57.937731 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:26:01.313844 mon.0 [INF] from='client.? 10.0.2.33:0/1014519' entity='client.admin' cmd=[{"prefix": "osd down", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:26:01.434547 mon.0 [INF] from='client.? 10.0.2.33:0/1014519' entity='client.admin' cmd='[{"prefix": "osd down", "ids": ["osd.4"]}]': finished

2015-09-14 13:26:01.452332 mon.0 [INF] osdmap e30: 4 osds: 0 up, 0 in

2015-09-14 13:26:01.477136 mon.0 [INF] pgmap v1656: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:26:04.414501 mon.0 [INF] from='client.? 10.0.2.33:0/1014552' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:26:07.547864 mon.0 [INF] from='client.? 10.0.2.33:0/1014585' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:26:07.675705 mon.0 [INF] from='client.? 10.0.2.33:0/1014585' entity='client.admin' cmd='[{"prefix": "osd rm", "ids": ["osd.4"]}]': finished

2015-09-14 13:26:07.693444 mon.0 [INF] osdmap e31: 3 osds: 0 up, 0 in

2015-09-14 13:26:07.718181 mon.0 [INF] pgmap v1657: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail