Need 4 nodes.
ceph-depoly
ceph1 <==> node1 (MDS: Gateway server)
ceph2 <==> node2
ceph3 <==> node3
cephc : ceph client node
hardware spec:
All nodes: need outside network and inside network
OSD nodes: need extra disk for OSD device
All server need outside network for install ceph.
In All Servers:
# useradd ceph
# passwd ceph
# yum install ntp ntpdate ntp-doc
In ceph-deploy server
# sudo yum install yum-plugin-priorities
# echo "ceph ALL = (root) NOPASSWD:ALL" | tee /etc/sudoers.d/ceph
# chmod 0440 /etc/sudoers.d/ceph
# scp /etc/sudoers.d/ceph 10.4.0.102:/etc/sudoers.d/ceph
# scp /etc/sudoers.d/ceph 10.4.0.103:/etc/sudoers.d/ceph
# scp /etc/sudoers.d/ceph 10.4.0.104:/etc/sudoers.d/ceph
# sudo vi /etc/yum.repos.d/ceph.repo
[ceph-noarch]
name=Ceph noarch packages
enabled=1
gpgcheck=1
type=rpm-md
Change account to ceph account:
$ ssh-keygen -t rsa
$ cd .ssh/
$ cp id_rsa.pub authorized_keys
$ cd -
$ ssh-copy-id ceph@node1
$ ssh-copy-id ceph@node2
$ ssh-copy-id ceph@node3
$ sudo yum install ceph-deploy
$ mkdir my-cluster
$ cd my-cluster/
$ ceph-deploy purge node1 node2 node3
$ ceph-deploy purgedata node1 node2 node3
$ ceph-deploy forgetkeys
$ rm -f ceph.*
Create cluster:
$ ceph-deploy new node1 <== create ceph.conf file
(or ceph-deploy new --cluster-network=11.130.1.0/24 --public-network=11.130.1.0/24 <hostname> )
<< Config ceph.conf file for your hardware environments >>
Install ceph packages
$ ceph-deploy install node1 node2 node3
Deploy the initial monitor and gather the keys:
$ ceph-deploy mon create-initial
Copy config and admin key to all ceph servers:
$ ceph-deploy admin node1 node2 node3 <== copy ceph config and ceph.client.admin.keyring
Add OSD device (/dev/sdb)
$ ceph-deploy osd create node1:sdb node2:sdb node3:sdb
Health check:
$ ssh node1 sudo ceph health
HEALTH_OK
$ ssh node1 sudo ceph -s
cluster e69ddb88-bef7-4fbe-9d41-8644032b40b2
health HEALTH_OK
monmap e1: 1 mons at {node1=10.4.0.102:6789/0}
election epoch 3, quorum 0 node1
osdmap e14: 3 osds: 3 up, 3 in
flags sortbitwise,require_jewel_osds
pgmap v24: 64 pgs, 1 pools, 0 bytes data, 0 objects
100 MB used, 76658 MB / 76759 MB avail
64 active+clean
Create Metadata server:
$ ceph-deploy mds create node1
Add more monitor server to storage server for high availability. ?????
$ ceph-deploy mon add node2
$ ceph-deploy mon add node3
$ ssh node1 sudo ceph -s
cluster e69ddb88-bef7-4fbe-9d41-8644032b40b2
health HEALTH_OK
monmap e1: 1 mons at {node1=10.4.0.102:6789/0}
election epoch 3, quorum 0 node1
osdmap e14: 3 osds: 3 up, 3 in
flags sortbitwise,require_jewel_osds
pgmap v24: 64 pgs, 1 pools, 0 bytes data, 0 objects
100 MB used, 76658 MB / 76759 MB avail
64 active+clean
Create RGW at gateway server:
$ ceph-deploy rgw create node1
Connect client node(cephc):
on admin-node for deploy:
# copy ceph account, ssh, password, sudoers
admin-node$ ceph-deploy install cephc
admin-node$ ceph-deploy admin cephc <== copy ceph config and ceph.client.admin.keyring
On Client node for making block device:
[root@cephc ~]# chmod 644 /etc/ceph/ceph.client.admin.keyring
[root@cephc ~]# rbd create disk01 --size 40960 <== rbd device size
[root@cephc ~]# rbd ls -l
NAME SIZE PARENT FMT PROT LOCK
disk01 40960M 2
foo 4096M 2
[root@cephc ~]# modprobe rbd
[root@cephc ~]# sudo rbd feature disable disk01 exclusive-lock object-map fast-diff deep-flatten
[root@cephc ~]# rbd map disk01
/dev/rbd0
[root@cephc ~]# rbd showmapped
id pool image snap device
0 rbd disk01 - /dev/rbd0
[root@cephc ~]# mkfs.xfs /dev/rbd0
[root@cephc ~]# mkdir /mnt/cephdisk
[root@cephc ~]# mount /dev/rbd0 /mnt/cephdisk/
[root@cephc ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 65G 2.1G 60G 4% /
devtmpfs 487M 0 487M 0% /dev
tmpfs 497M 0 497M 0% /dev/shm
tmpfs 497M 6.6M 490M 2% /run
tmpfs 497M 0 497M 0% /sys/fs/cgroup
/dev/sda1 477M 98M 354M 22% /boot
tmpfs 100M 0 100M 0% /run/user/0
/dev/rbd0 40G 33M 40G 1% /mnt/cephdisk
RBD mount script
---------------------------------------------------------------------------------------------------
# Change with your pools name
export poolname=rbd
# CHange with your disk image name
export rbdimage=disk01
# Mount Directory
export mountpoint=/mnt/mydisk
# Image mount/unmount and pool are passed from the systems service as arguments
# Determine if we are mounting or unmounting
if [ "$1" == "m" ]; then
modprobe rbd
rbd feature disable $rbdimage exclusive-lock object-map fast-diff deep-flatten
rbd map $rbdimage --id admin --keyring /etc/ceph/ceph.client.admin.keyring
mkdir -p $mountpoint
mount /dev/rbd/$poolname/$rbdimage $mountpoint
fi
if [ "$1" == "u" ]; then
umount $mountpoint
rbd unmap /dev/rbd/$poolname/$rbdimage
fi
---------------------------------------------------------------------------------------------------
Testing with single network:
[root@cephc ~]# cd /mnt/cephdisk/
[root@cephc cephdisk]# ls
[root@cephc cephdisk]# dd if=/dev/zero of=20G bs=1M count=20480
20480+0 records in
20480+0 records out
21474836480 bytes (21 GB) copied, 814.81 s, 26.4 MB/s
[root@cephc cephdisk]#
[root@cephc cephdisk]# time rm -f 20G
real 0m0.132s
user 0m0.001s
sys 0m0.110s
[root@cephc cephdisk]# dd if=/dev/zero of=2G bs=1M count=2048
2048+0 records in
2048+0 records out
2147483648 bytes (2.1 GB) copied, 80.6485 s, 26.6 MB/s
Testing with dual network:
Copy /etc/ceph/ceph.conf file to OSD servers (node1,node2,node3)
and reboot node1,2,3 server for apply it.
[root@node1 ~]# cat /etc/ceph/ceph.conf
[global]
fsid = e69ddb88-bef7-4fbe-9d41-8644032b40b2
mon_initial_members = node1
mon_host = 10.4.0.102
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
[osd]
public_network = 10.4.0.0/16
cluster_network = 10.5.0.0/16
[osd.0]
public_addr = 10.4.0.102:6801
cluster_addr = 10.5.0.1
[osd.1]
public_addr = 10.4.0.103:6802
cluster_addr = 10.5.0.2
[osd.2]
public_addr = 10.4.0.104:6803
cluster_addr = 10.5.0.3
[ceph@node1 ~]$ sudo ceph auth list
installed auth entries:
mds.node1
key: AQBi7lNa8E3rJRAA8h6VKiIrT3Jjq2QHBlktmw==
caps: [mds] allow
caps: [mon] allow profile mds
caps: [osd] allow rwx
osd.0
key: AQDF7VNaPpZXChAAoF/ppU5TLPV0aWgj/lL8Cg==
caps: [mon] allow profile osd
caps: [osd] allow *
osd.1
key: AQDR7VNa55uGNxAAOIXPeXPJNihYILKfVqQmew==
caps: [mon] allow profile osd
caps: [osd] allow *
osd.2
key: AQDg7VNa6HKRGhAADWqYn32xh7g1LPEkeTIBFw==
caps: [mon] allow profile osd
caps: [osd] allow *
client.admin
key: AQBD7VNaej9jCRAAaNWDjIK7KTUsuY90lIqtwg==
caps: [mds] allow *
caps: [mon] allow *
caps: [osd] allow *
client.bootstrap-mds
key: AQBD7VNaXUrtGRAAuQGP7ElLiMlWExWE2PM2iQ==
caps: [mon] allow profile bootstrap-mds
client.bootstrap-osd
key: AQBD7VNa/BftDhAA0grK5NqxJf2zZAXj61158Q==
caps: [mon] allow profile bootstrap-osd
client.bootstrap-rgw
key: AQBD7VNa46ZtFBAA1JSr5c4Jt+BqHIhLN1I9wA==
caps: [mon] allow profile bootstrap-rgw
client.rgw.node1
key: AQCh71Na4bLZAhAAnlP+2BGA8kK80FoiEEklIw==
caps: [mon] allow rw
caps: [osd] allow rwx
[ceph@node1 ~]$ sudo rados df
pool name KB objects clones degraded unfound rd rd KB wr wr KB
.rgw.root 2 4 0 0 0 48 37 4 5
default.rgw.control 0 8 0 0 0 0 0 0 0
default.rgw.data.root 0 0 0 0 0 0 0 0 0
default.rgw.gc 0 32 0 0 0 288 256 192 0
default.rgw.log 0 127 0 0 0 6096 5969 4064 0
default.rgw.users.uid 0 0 0 0 0 0 0 0 0
rbd 22163889 5435 0 0 0 473 6140 55750 33583613
total used 66644872 5606
total avail 11956808
total space 78601680
above command reference: http://docs.ceph.com/docs/master/rados/operations/pools/#create-a-pool
[root@cephc cephdisk]# dd if=/dev/zero of=2G3 bs=1M count=2048
2048+0 records in
2048+0 records out
2147483648 bytes (2.1 GB) copied, 73.7127 s, 29.1 MB/s
[root@cephc cephdisk]# dd if=/dev/zero of=2G4 bs=1M count=2048
2048+0 records in
2048+0 records out
2147483648 bytes (2.1 GB) copied, 76.5691 s, 28.0 MB/s
[root@cephc cephdisk]# dd if=/dev/zero of=2G5 bs=1M count=2048
2048+0 records in
2048+0 records out
Reboot client node:
[root@cephc ~]# ls /dev/rbd*
ls: cannot access /dev/rbd*: No such file or directory
[root@cephc ~]# modprobe rbd
[root@cephc ~]# lsmod |grep rbd
rbd 83889 0
libceph 282661 1 rbd
[root@cephc ~]# ls /dev/rbd*
ls: cannot access /dev/rbd*: No such file or directory
[root@cephc ~]# rbd ls -l
NAME SIZE PARENT FMT PROT LOCK
disk01 40960M 2
foo 4096M 2
[root@cephc ~]# rbd feature disable disk01 exclusive-lock object-map fast-diff deep-flatten
rbd: failed to update image features: (22) Invalid argument
2018-01-09 00:48:23.297393 7f9028a25d80 -1 librbd: one or more requested features are already disabled
[root@cephc ~]# rbd map disk01
/dev/rbd0
[root@cephc ~]# ls /dev/rbd*
/dev/rbd0
/dev/rbd:
rbd
[root@cephc ~]# mount /dev/rbd0 /mnt/cephdisk
[root@cephc ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 65G 2.1G 60G 4% /
devtmpfs 487M 0 487M 0% /dev
tmpfs 497M 0 497M 0% /dev/shm
tmpfs 497M 6.6M 490M 2% /run
tmpfs 497M 0 497M 0% /sys/fs/cgroup
/dev/sda1 477M 98M 354M 22% /boot
tmpfs 100M 0 100M 0% /run/user/0
/dev/rbd0 40G 11G 30G 26% /mnt/cephdisk
[root@cephc ~]# ls -lh /mnt/cephdisk/
total 10G
-rw-r--r-- 1 root root 2.0G Jan 9 00:08 2G
-rw-r--r-- 1 root root 2.0G Jan 9 00:27 2G2nd
-rw-r--r-- 1 root root 2.0G Jan 9 00:38 2G3
-rw-r--r-- 1 root root 2.0G Jan 9 00:40 2G4
-rw-r--r-- 1 root root 2.0G Jan 9 00:42 2G5