centos7单节点部署ceph（mon/mgr/osd/mgr/rgw）

使用ceph建议采用多节点多磁盘方式部署，本文章仅作为单节点部署参考，请勿用于生产环境

yum 配置文件添加如下相关源

[root@testserver01 ~]# cat /etc/yum.conf

[epel]

name=CentOS7-epel

baseurl=https://mirrors.aliyun.com/epel/7/x86_64/

gpgcheck=0

enabled=1

[base]

name=CentOS7-OS

baseurl=https://mirrors.aliyun.com/centos/7/os/x86_64/

gpgcheck=0

enabled=1

[nautilus-x86_64]

name=ceph-x86_64

baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/x86_64/

gpgcheck=0

enabled=1

[nautilus-noarch]

name=ceph-noarch

baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/

gpgcheck=0

enabled=1

安装ceph mgr依赖的python2-Werkzeug包

[root@testserver01 ~]# rpm -ivh http://rpmfind.net/linux/opensuse/distribution/leap/15.3/repo/oss/noarch/python2-Werkzeug-1.0.1-1.10.noarch.rpm

[root@testserver01 ~]# yum install ceph chrony gdisk ceph-radosgw ceph-deploy ceph-mgr-dashboard -y
修改系统参数

[root@testserver01 ~]# sysctl -w kernel.pid_max=4194303; echo 'kernel.pid_max=4194303' >> /etc/sysctl.conf; sysctl -p

添加ceph配置文件

[root@testserver01 ~]# cat /etc/ceph/ceph.conf

[global]

fsid = b080bada-7f10-11ee-8f11-666666666666

mon initial members = testserver01

mon host = 192.168.1.128

public network = 192.168.1.0/24

auth cluster required = cephx

auth service required = cephx

auth client required = cephx

osd pool default pg num = 16

osd pool default pgp num = 16

osd crush update on start=false

osd_pool_default_min_size = 1

mon_osd_down_out_subtree_limit = host

osd_pool_default_size = 1

mon_warn_on_pool_no_redundancy = false

mon_warn_on_insecure_global_id_reclaim_allowed = false

[osd]

osd_scrub_begin_hour = 23

osd_scrub_end_hour = 7

部署mon

[root@testserver01 ~]# ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'

creating /tmp/ceph.mon.keyring

[root@testserver01 ~]# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'

creating /etc/ceph/ceph.client.admin.keyring

[root@testserver01 ~]# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring

importing contents of /etc/ceph/ceph.client.admin.keyring into /tmp/ceph.mon.keyring

[root@testserver01 ~]# monmaptool --create --add testserver01 192.168.1.128 --fsid b080bada-7f10-11ee-8f11-666666666666 /tmp/monmap

monmaptool: monmap file /tmp/monmap

monmaptool: set fsid to b080bada-7f10-11ee-8f11-666666666666

monmaptool: writing epoch 0 to /tmp/monmap (1 monitors)

[root@testserver01 ~]# ceph-mon --mkfs -i testserver01 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

[root@testserver01 ~]# chown ceph.ceph /var/lib/ceph/mon -R

[root@testserver01 ~]# systemctl start ceph-mon@testserver01

system[root@testserver01 ~]# systemctl enable ceph-mon@testserver01

Created symlink from /etc/systemd/system/ceph-mon.target.wants/ceph-mon@testserver01.service to /usr/lib/systemd/system/ceph-mon@.service.

[root@testserver01 ~]# ceph mon enable-msgr2

[root@testserver01 ~]# ceph -s

cluster:

id: b080bada-7f10-11ee-8f11-666666666666

health: HEALTH_OK

services:

mon: 1 daemons, quorum testserver01 (age 43s)

mgr: no daemons active

osd: 0 osds: 0 up, 0 in

data:

pools: 0 pools, 0 pgs

objects: 0 objects, 0 B

usage: 0 B used, 0 B / 0 B avail

pgs:

部署mgr

[root@testserver01 ~]# mkdir -p /var/lib/ceph/mgr/ceph-testserver01

[root@testserver01 ~]# ceph auth get-or-create mgr.testserver01 mon 'allow profile mgr' osd 'allow *' mds 'allow *' >/var/lib/ceph/mgr/ceph-testserver01/keyring

[root@testserver01 ~]# chown ceph.ceph -R /var/lib/ceph/mgr/

[root@testserver01 ~]# systemctl start ceph-mgr@testserver01

[root@testserver01 ~]# systemctl enable ceph-mgr@testserver01

Created symlink from /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@testserver01.service to /usr/lib/systemd/system/ceph-mgr@.service.

[root@testserver01 ~]# ceph -s

cluster:

id: b080bada-7f10-11ee-8f11-666666666666

health: HEALTH_WARN

OSD count 0 < osd_pool_default_size 1

services:

mon: 1 daemons, quorum testserver01 (age 4m)

mgr: testserver01(active, since 70s)

osd: 0 osds: 0 up, 0 in

data:

pools: 0 pools, 0 pgs

objects: 0 objects, 0 B

usage: 0 B used, 0 B / 0 B avail

pgs:

部署osd

[root@testserver01 ~]# ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring

exported keyring for client.bootstrap-osd

[root@testserver01 ~]# sgdisk --zap-all /dev/vdb

Creating new GPT entries.

GPT data structures destroyed! You may now partition the disk using fdisk or

other utilities.

[root@testserver01 ~]# ceph-volume lvm create --bluestore --data /dev/vdb

配置crushmap

[root@testserver01 ~]# ceph osd crush move osd.0 host=testserver01 root=default

moved item id 0 name 'osd.0' to location {host=testserver01,root=default} in crush map

[root@testserver01 ~]# ceph osd crush reweight osd.0 0.01

reweighted item id 0 name 'osd.0' to 0.01 in crush map

创建块存储池

[root@testserver01 ~]# ceph osd pool create rbdpool01 32 32

pool 'rbdpool01' created

[root@testserver01 ~]# ceph osd pool application enable rbdpool01 rbd

enabled application 'rbd' on pool 'rbdpool01'

[root@testserver01 ~]# ceph osd pool ls detail

pool 1 'rbdpool01' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 10 flags hashpspool stripe_width 0 application rbd

创建rbd存储块

[root@testserver01 ~]# rbd create rbdpool01/testimg --size 5G

[root@testserver01 ~]# rbd ls -p rbdpool01 -l

NAME SIZE PARENT FMT PROT LOCK

testimg 5 GiB 2

添加mds服务

[root@testserver01 ~]# mkdir -p /var/lib/ceph/mds/ceph-testserver01

[root@testserver01 ~]# ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-testserver01/keyring --gen-key -n mds.testserver01

creating /var/lib/ceph/mds/ceph-testserver01/keyring

[root@testserver01 ~]# ceph auth add mds.testserver01 osd 'allow rwx' mds 'allow *' mon 'allow profile mds' -i /var/lib/ceph/mds/ceph-testserver01/keyring

added key for mds.testserver01

[root@testserver01 ~]# chown -R ceph.ceph /var/lib/ceph/

[root@testserver01 ~]# systemctl enable ceph-mds@testserver01

Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@testserver01.service to /usr/lib/systemd/system/ceph-mds@.service.

[root@testserver01 ~]# systemctl start ceph-mds@testserver01

添加cephfs存储池

[root@testserver01 ~]# ceph osd pool create cephfs-metadata 16 16

pool 'cephfs-metadata' created

[root@testserver01 ~]# ceph osd pool create cephfs-data 32 32

pool 'cephfs-data' created

[root@testserver01 ~]# ceph fs new cephfs cephfs-metadata cephfs-data

new fs with metadata pool 2 and data pool 3

[root@testserver01 ~]# ceph fs status

cephfs - 0 clients

======

+------+--------+--------------+---------------+-------+-------+

+------+--------+--------------+---------------+-------+-------+

| 0 | active | testserver01 | Reqs: 0 /s | 10 | 13 |

+------+--------+--------------+---------------+-------+-------+

+-----------------+----------+-------+-------+

+-----------------+----------+-------+-------+

| cephfs-metadata | metadata | 512k | 8697M |

| cephfs-data | data | 0 | 8697M |

+-----------------+----------+-------+-------+

+-------------+

| Standby MDS |

+-------------+

MDS version: ceph version 14.2.22 (ca74598065096e6fcbd8433c8779a2be0c889351) nautilus (stable)

[root@testserver01 ~]#

部署rgw (默认7480端口)

[root@testserver01 ~]# cp /etc/ceph/ceph.conf /tmp/

[root@testserver01 ~]# cd /tmp

[root@testserver01 tmp]# ceph auth get client.bootstrap-rgw -o ceph.bootstrap-rgw.keyring

exported keyring for client.bootstrap-rgw

[root@testserver01 tmp]# ceph-deploy --overwrite-conf rgw create testserver01

[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf

[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy --overwrite-conf rgw create testserver01

[ceph_deploy.cli][INFO ] ceph-deploy options:

[ceph_deploy.cli][INFO ] username : None

[ceph_deploy.cli][INFO ] verbose : False

[ceph_deploy.cli][INFO ] rgw : [('testserver01', 'rgw.testserver01')]

[ceph_deploy.cli][INFO ] overwrite_conf : True

[ceph_deploy.cli][INFO ] subcommand : create

[ceph_deploy.cli][INFO ] quiet : False

[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fca20b03a28>

[ceph_deploy.cli][INFO ] cluster : ceph

[ceph_deploy.cli][INFO ] func : <function rgw at 0x7fca21148050>

[ceph_deploy.cli][INFO ] ceph_conf : None

[ceph_deploy.cli][INFO ] default_release : False

[ceph_deploy.rgw][DEBUG ] Deploying rgw, cluster ceph hosts testserver01:rgw.testserver01

[testserver01][DEBUG ] connected to host: testserver01

[testserver01][DEBUG ] detect platform information from remote host

[testserver01][DEBUG ] detect machine type

[ceph_deploy.rgw][INFO ] Distro info: CentOS Linux 7.9.2009 Core

[ceph_deploy.rgw][DEBUG ] remote host will use systemd

[ceph_deploy.rgw][DEBUG ] deploying rgw bootstrap to testserver01

[testserver01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf

[testserver01][WARNIN] rgw keyring does not exist yet, creating one

[testserver01][DEBUG ] create a keyring file

[testserver01][DEBUG ] create path recursively if it doesn't exist

[testserver01][INFO ] Running command: ceph --cluster ceph --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/ceph.keyring auth get-or-create client.rgw.testserver01 osd allow rwx mon allow rw -o /var/lib/ceph/radosgw/ceph-rgw.testserver01/keyring

[testserver01][INFO ] Running command: systemctl enable ceph-radosgw@rgw.testserver01

[testserver01][WARNIN] Created symlink from /etc/systemd/system/ceph-radosgw.target.wants/ceph-radosgw@rgw.testserver01.service to /usr/lib/systemd/system/ceph-radosgw@.service.

[testserver01][INFO ] Running command: systemctl start ceph-radosgw@rgw.testserver01

[testserver01][INFO ] Running command: systemctl enable ceph.target

[ceph_deploy.rgw][INFO ] The Ceph Object Gateway (RGW) is now running on host testserver01 and default port 7480

[root@testserver01 tmp]# radosgw-admin user create --uid=admin --display-name="admin user" --system

{

"user_id": "admin",

"display_name": "admin user",

"email": "",

"suspended": 0,

"max_buckets": 1000,

"subusers": [],

"keys": [

{

"user": "admin",

"access_key": "7DRHRKOVTQF0EBWHL7LZ",

"secret_key": "01qnDj58DmWTjQZzCwDv9SLfC6x2bIsWgD2EgHri"

}

"swift_keys": [],

"caps": [],

"op_mask": "read, write, delete",

"system": "true",

"default_placement": "",

"default_storage_class": "",

"placement_tags": [],

"bucket_quota": {

"enabled": false,

"check_on_raw": false,

"max_size": -1,

"max_size_kb": 0,

"max_objects": -1

"user_quota": {

"enabled": false,

"check_on_raw": false,

"max_size": -1,

"max_size_kb": 0,

"max_objects": -1

"temp_url_keys": [],

"type": "rgw",

"mfa_ids": []

}

开启并配置mgr dashboard

[root@testserver01 ~]# ceph mgr module enable prometheus (开启监控端口，9283端口，需要安装ceph-mgr-dashboard包)

[root@testserver01 ~]# ceph config set mgr mgr/dashboard/ssl false

[root@testserver01 ~]# ceph mgr module enable dashboard (关闭dashboard ssl，默认8080端口)

[root@testserver01 tmp]# echo 'passwordxxx' > password.txt

[root@testserver01 tmp]# ceph dashboard ac-user-create admin administrator -i password.txt

{"username": "admin", "lastUpdate": 1709353417, "name": null, "roles": ["administrator"], "password": "$2b $12$ FcEB.KYWVB8BCXaXxu5VJOQbUqwO0KCUOEfi2wgSnmlFPW3RqTQp2", "email": null}

监控端口：

访问mgr dashboard http://192.168.1.128:8080

配置rgw dashboard

[root@testserver01 tmp]# radosgw-admin user info --uid=admin (查看ak和sk)

[root@testserver01 tmp]# cat /tmp/a.txt

7DRHRKOVTQF0EBWHL7LZ

[root@testserver01 tmp]# cat /tmp/s.txt

01qnDj58DmWTjQZzCwDv9SLfC6x2bIsWgD2EgHri

[root@testserver01 tmp]# ceph dashboard set-rgw-api-access-key -i a.txt

Option RGW_API_ACCESS_KEY updated

[root@testserver01 tmp]# ceph dashboard set-rgw-api-secret-key -i s.txt

Option RGW_API_SECRET_KEY updated

查看健康状态，osd，pool，crush rule，端口等

[root@testserver01 ~]# ceph -s

cluster:

id: b080bada-7f10-11ee-8f11-666666666666

health: HEALTH_OK

services:

mon: 1 daemons, quorum testserver01 (age 33m)

mgr: testserver01(active, since 11m)

mds: cephfs:1 {0=testserver01=up:active}

osd: 1 osds: 1 up (since 28m), 1 in (since 28m)

rgw: 1 daemon active (testserver01)

task status:

data:

pools: 7 pools, 144 pgs

objects: 216 objects, 5.7 KiB

usage: 1.0 GiB used, 9.0 GiB / 10 GiB avail

pgs: 144 active+clean

[root@testserver01 ~]# ceph osd df

ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS

0 hdd 0.00999 1.00000 10 GiB 1.0 GiB 4.9 MiB 0 B 1 GiB 9.0 GiB 10.05 1.00 144 up

TOTAL 10 GiB 1.0 GiB 4.9 MiB 0 B 1 GiB 9.0 GiB 10.05

MIN/MAX VAR: 1.00/1.00 STDDEV: 0

[root@testserver01 ~]# ceph osd pool ls detail

pool 1 'rbdpool01' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 28 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd

removed_snaps [1~3]

pool 2 'cephfs-metadata' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 17 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs

pool 3 'cephfs-data' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 17 flags hashpspool stripe_width 0 application cephfs

pool 4 '.rgw.root' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 20 flags hashpspool stripe_width 0 application rgw

pool 5 'default.rgw.control' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 22 flags hashpspool stripe_width 0 application rgw

pool 6 'default.rgw.meta' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 24 flags hashpspool stripe_width 0 application rgw

pool 7 'default.rgw.log' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 26 flags hashpspool stripe_width 0 application rgw

[root@testserver01 ~]# ceph osd crush rule dump

[

{

"rule_id": 0,

"rule_name": "replicated_rule",

"ruleset": 0,

"type": 1,

"min_size": 1,

"max_size": 10,

"steps": [

{

"op": "take",

"item": -1,

"item_name": "default"

{

"op": "chooseleaf_firstn",

"num": 0,

"type": "host"

{

"op": "emit"

}

]

}

]

[root@testserver01 ~]# netstat -tnlp

Active Internet connections (only servers)

Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name

tcp 0 0 192.168.1.128:6809 0.0.0.0:* LISTEN 2694/ceph-osd

tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 977/master

tcp 0 0 192.168.1.128:6810 0.0.0.0:* LISTEN 3037/ceph-mds

tcp 0 0 192.168.1.128:6811 0.0.0.0:* LISTEN 3037/ceph-mds

tcp 0 0 192.168.1.128:3300 0.0.0.0:* LISTEN 1944/ceph-mon

tcp 0 0 192.168.1.128:6789 0.0.0.0:* LISTEN 1944/ceph-mon

tcp 0 0 192.168.1.128:6800 0.0.0.0:* LISTEN 4480/ceph-mgr

tcp 0 0 192.168.1.128:6801 0.0.0.0:* LISTEN 4480/ceph-mgr

tcp 0 0 192.168.1.128:6802 0.0.0.0:* LISTEN 2694/ceph-osd

tcp 0 0 192.168.1.128:6803 0.0.0.0:* LISTEN 2694/ceph-osd

tcp 0 0 192.168.1.128:6804 0.0.0.0:* LISTEN 2694/ceph-osd

tcp 0 0 192.168.1.128:6805 0.0.0.0:* LISTEN 2694/ceph-osd

tcp 0 0 192.168.1.128:6806 0.0.0.0:* LISTEN 2694/ceph-osd

tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 873/sshd

tcp 0 0 192.168.1.128:6807 0.0.0.0:* LISTEN 2694/ceph-osd

tcp 0 0 0.0.0.0:7480 0.0.0.0:* LISTEN 3280/radosgw

tcp 0 0 192.168.1.128:6808 0.0.0.0:* LISTEN 2694/ceph-osd

tcp6 0 0 ::1:25 :::* LISTEN 977/master

tcp6 0 0 :::9283 :::* LISTEN 4480/ceph-mgr

tcp6 0 0 :::8080 :::* LISTEN 4480/ceph-mgr

tcp6 0 0 :::22 :::* LISTEN 873/sshd

tcp6 0 0 :::7480 :::* LISTEN 3280/radosgw