ceph安装过程

创建群集

[2019-03-20 18:35:04,232][ceph_deploy.conf][DEBUG ] found configuration file at: /home/sceph/.cephdeploy.conf

[2019-03-20 18:35:04,233][ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy new ceph1

sudo /usr/sbin/ip link show

sudo /usr/sbin/ip addr show

========================================

安装ceph文件

[2019-03-21 10:36:07,664][ceph_deploy.conf][DEBUG ] found configuration file at: /home/sceph/.cephdeploy.conf

[2019-03-21 10:36:07,666][ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy install ceph1 ceph2 ceph3

sudo yum clean all

sudo yum -y install epel-release

sudo yum -y install yum-plugin-priorities

sudo rpm --import https://download.ceph.com/keys/release.asc

sudo yum remove -y ceph-release

sudo yum install -y https://download.ceph.com/rpm-mimic/el7/noarch/ceph-release-1-0.el7.noarch.rpm

ensuring that /etc/yum.repos.d/ceph.repo contains a high priority

sudo yum -y install ceph ceph-radosgw

[2019-03-21 11:15:01,990][ceph1][INFO ] Running command: sudo ceph --version

[2019-03-21 11:15:02,126][ceph1][DEBUG ] ceph version 13.2.5 (cbff874f9007f1869bfd3821b7e33b2a6ffd4988) mimic (stable)

====================================

创建mon

[2019-03-21 15:29:30,161][ceph_deploy.conf][DEBUG ] found configuration file at: /home/sceph/.cephdeploy.conf

[2019-03-21 15:29:30,161][ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mon create-initial

sudo ceph-mon --cluster ceph --mkfs -i ceph1 --keyring /var/lib/ceph/tmp/ceph-ceph1.mon.keyring --setuser 167 --setgroup 167

sudo systemctl enable ceph.target

sudo systemctl enable ceph-mon@ceph1

sudo systemctl start ceph-mon@ceph1

sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph1.asok mon_status

sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.ceph1.asok mon_status

sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph1/keyring auth get client.admin

sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph1/keyring auth get client.bootstrap-mds

sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph1/keyring auth get client.bootstrap-mgr

sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph1/keyring auth get client.bootstrap-osd

sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph1/keyring auth get client.bootstrap-rgw

================================

推送配置

[2019-03-21 17:22:03,950][ceph_deploy.conf][DEBUG ] found configuration file at: /home/sceph/.cephdeploy.conf

[2019-03-21 17:22:03,950][ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy admin ceph1 ceph2 ceph3

Pushing admin keys and conf to ceph3

创建mgr

[2019-03-21 17:26:53,722][ceph_deploy.conf][DEBUG ] found configuration file at: /home/sceph/.cephdeploy.conf

[2019-03-21 17:26:53,723][ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mgr create ceph2

Deploying mgr, cluster ceph hosts ceph2:ceph2

sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.ceph2 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-ceph2/keyring

sudo systemctl enable ceph-mgr@ceph2

创建三个osd，在admin node上

ceph-deploy osd create --data /dev/vdb ceph1

ceph-deploy osd create --data /dev/vdb ceph2

ceph-deploy osd create --data /dev/vdb ceph3

到此完毕，检查状态是否正常

sudo ceph health

sudo ceph -s

ceph quorum_status --format json-pretty

实践，创建pool，并存储一个对象

echo "11111" > my.txt

ceph osd pool create mypool 8

rados put m1 my.txt --pool=mypool

rados -p mypool ls

rados df

检查到底存了没有

[root@ceph1 ~]# rados -p mypool ls

m1

[root@ceph1 ~]# rados df

POOL_NAME USED OBJECTS CLONES COPIES MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS RD WR_OPS WR

mypool 12 B 1 0 3 0 0 0 0 0 B 1 1 KiB

total_objects 1

total_used 3.0 GiB

total_avail 177 GiB

total_space 180 GiB

[root@ceph1 ~]# cat my.txt

11111111111

标识对象位置

[root@ceph1 ~]# ceph osd map mypool m1

osdmap e16 pool 'mypool' (1) object 'm1' -> pg 1.c3958388 (1.0) -> up ([1,2,0], p1) acting ([1,2,0], p1)

[root@ceph1 ~]#

删除对象

[root@ceph1 ~]# rados rm m1 --pool=mypool

删除pool

[root@ceph1 ~]# ceph osd pool rm mypool

Error EPERM: WARNING: this will *PERMANENTLY DESTROY* all data stored in pool mypool. If you are *ABSOLUTELY CERTAIN* that is what you want, pass the pool name *twice*, followed by --yes-i-really-really-mean-it.

[root@ceph1 ~]# ceph osd pool rm mypool mypool --yes-i-really-really-mean-it

Error EPERM: pool deletion is disabled; you must first set the mon_allow_pool_delete config option to true before you can destroy a pool

==================================

ceph mgr module enable dashboard

ceph dashboard create-self-signed-cert

禁用ssl

ceph config set mgr mgr/dashboard/ssl false

[root@ceph2 yum.repos.d]# ceph mgr services

{

"dashboard": "https://192.168.7.152:9800/"

}

===================================

[sceph@ceph1 ~]$ ceph-deploy mds create ceph2

=====================================

[sceph@ceph1 ~]$ ceph-deploy --overwrite-conf rgw create ceph1

radosgw-admin user create --uid=786 --display-name=fff --system

radosgw-admin user info --uid=786

ceph dashboard set-rgw-api-access-key Y2CAASJE4Z0HFAD26R1R

ceph dashboard set-rgw-api-secret-key dJzSwCHeMkqhbksV0zcDnalPzBPjHz3wRG3aRlSN

然后在dashboard上就可看到object gateway了

[root@ceph1 ceph]# systemctl status ceph-radosgw@ceph1

vi /etc/ceph/ceph.conf

[client.rgw.ceph1]

rgw_frontends = civetweb port=9080

systemctl restart ceph-radosgw.target

systemctl -a

#ceph-deploy --overwrite-conf config push ceph1

=======================================

mon的数据库是rocksdb

https://github.com/facebook/rocksdb/wiki/Administration-and-Data-Access-Tool

在/var/lib/ceph/mon/ceph-ceph1/store.db

59 unzip rocksdb-master.zip

61 cd rocksdb-master

65 yum install gcc gcc-c++ perl

70 yum install perl

72 make

73 make install

83 scp -r 192.168.7.151:/var/lib/ceph/mon/ceph-ceph1/store.db ~/

87 cd ~/store.db/

90 ~/rocksdb/rocksdb-master/ldb --db=007031.sst scan

91 ~/rocksdb/rocksdb-master/sst_dump -h

92 ~/rocksdb/rocksdb-master/sst_dump --file=007031.sst --command=scan

93 ~/rocksdb/rocksdb-master/sst_dump --file=007031.sst --command=scan --read_num=5

97 ~/rocksdb/rocksdb-master/sst_dump --file=007031.sst --command=raw

99 less 007031_dump.txt