centos7单节点部署ceph（mon/mgr/osd/mgr/rgw）

使用ceph建议采用多节点多磁盘方式部署，本文章仅作为单节点部署参考，请勿用于生产环境

yum 配置文件添加如下相关源

root@testserver01 \~\]# cat /etc/yum.conf \[epel

name=CentOS7-epel

baseurl=https://mirrors.aliyun.com/epel/7/x86_64/

gpgcheck=0

enabled=1

base

name=CentOS7-OS

baseurl=https://mirrors.aliyun.com/centos/7/os/x86_64/

gpgcheck=0

enabled=1

nautilus-x86_64

name=ceph-x86_64

baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/x86_64/

gpgcheck=0

enabled=1

nautilus-noarch

name=ceph-noarch

baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/

gpgcheck=0

enabled=1

安装ceph mgr依赖的python2-Werkzeug包

root@testserver01 \~\]# rpm -ivh http://rpmfind.net/linux/opensuse/distribution/leap/15.3/repo/oss/noarch/python2-Werkzeug-1.0.1-1.10.noarch.rpm 3. \[root@testserver01 \~\]# yum install ceph chrony gdisk ceph-radosgw ceph-deploy ceph-mgr-dashboard -y 4. 修改系统参数 \[root@testserver01 \~\]# sysctl -w kernel.pid_max=4194303; echo 'kernel.pid_max=4194303' \>\> /etc/sysctl.conf; sysctl -p 5. 添加ceph配置文件 \[root@testserver01 \~\]# cat /etc/ceph/ceph.conf \[global

fsid = b080bada-7f10-11ee-8f11-666666666666

mon initial members = testserver01

mon host = 192.168.1.128

public network = 192.168.1.0/24

auth cluster required = cephx

auth service required = cephx

auth client required = cephx

osd pool default pg num = 16

osd pool default pgp num = 16

osd crush update on start=false

osd_pool_default_min_size = 1

mon_osd_down_out_subtree_limit = host

osd_pool_default_size = 1

mon_warn_on_pool_no_redundancy = false

mon_warn_on_insecure_global_id_reclaim_allowed = false

osd

osd_scrub_begin_hour = 23

osd_scrub_end_hour = 7

部署mon

root@testserver01 \~\]# ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow \*' creating /tmp/ceph.mon.keyring \[root@testserver01 \~\]# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow \*' --cap osd 'allow \*' --cap mds 'allow \*' --cap mgr 'allow \*' creating /etc/ceph/ceph.client.admin.keyring \[root@testserver01 \~\]# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring importing contents of /etc/ceph/ceph.client.admin.keyring into /tmp/ceph.mon.keyring \[root@testserver01 \~\]# monmaptool --create --add testserver01 192.168.1.128 --fsid b080bada-7f10-11ee-8f11-666666666666 /tmp/monmap monmaptool: monmap file /tmp/monmap monmaptool: set fsid to b080bada-7f10-11ee-8f11-666666666666 monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) \[root@testserver01 \~\]# ceph-mon --mkfs -i testserver01 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring \[root@testserver01 \~\]# chown ceph.ceph /var/lib/ceph/mon -R \[root@testserver01 \~\]# systemctl start ceph-mon@testserver01 system\[root@testserver01 \~\]# systemctl enable ceph-mon@testserver01 Created symlink from /etc/systemd/system/ceph-mon.target.wants/[email protected] to /usr/lib/systemd/system/[email protected]. \[root@testserver01 \~\]# ceph mon enable-msgr2 \[root@testserver01 \~\]# ceph -s cluster: id: b080bada-7f10-11ee-8f11-666666666666 health: HEALTH_OK services: mon: 1 daemons, quorum testserver01 (age 43s) mgr: no daemons active osd: 0 osds: 0 up, 0 in data: pools: 0 pools, 0 pgs objects: 0 objects, 0 B usage: 0 B used, 0 B / 0 B avail pgs: 7. 部署mgr \[root@testserver01 \~\]# mkdir -p /var/lib/ceph/mgr/ceph-testserver01 \[root@testserver01 \~\]# ceph auth get-or-create mgr.testserver01 mon 'allow profile mgr' osd 'allow \*' mds 'allow \*' \>/var/lib/ceph/mgr/ceph-testserver01/keyring \[root@testserver01 \~\]# chown ceph.ceph -R /var/lib/ceph/mgr/ \[root@testserver01 \~\]# systemctl start ceph-mgr@testserver01 \[root@testserver01 \~\]# systemctl enable ceph-mgr@testserver01 Created symlink from /etc/systemd/system/ceph-mgr.target.wants/[email protected] to /usr/lib/systemd/system/[email protected]. \[root@testserver01 \~\]# ceph -s cluster: id: b080bada-7f10-11ee-8f11-666666666666 health: HEALTH_WARN OSD count 0 \< osd_pool_default_size 1 services: mon: 1 daemons, quorum testserver01 (age 4m) mgr: testserver01(active, since 70s) osd: 0 osds: 0 up, 0 in data: pools: 0 pools, 0 pgs objects: 0 objects, 0 B usage: 0 B used, 0 B / 0 B avail pgs: 8. 部署osd \[root@testserver01 \~\]# ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring exported keyring for client.bootstrap-osd \[root@testserver01 \~\]# sgdisk --zap-all /dev/vdb Creating new GPT entries. GPT data structures destroyed! You may now partition the disk using fdisk or other utilities. \[root@testserver01 \~\]# ceph-volume lvm create --bluestore --data /dev/vdb 9. 配置crushmap \[root@testserver01 \~\]# ceph osd crush move osd.0 host=testserver01 root=default moved item id 0 name 'osd.0' to location {host=testserver01,root=default} in crush map \[root@testserver01 \~\]# ceph osd crush reweight osd.0 0.01 reweighted item id 0 name 'osd.0' to 0.01 in crush map 10. 创建块存储池 \[root@testserver01 \~\]# ceph osd pool create rbdpool01 32 32 pool 'rbdpool01' created \[root@testserver01 \~\]# ceph osd pool application enable rbdpool01 rbd enabled application 'rbd' on pool 'rbdpool01' \[root@testserver01 \~\]# ceph osd pool ls detail pool 1 'rbdpool01' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 10 flags hashpspool stripe_width 0 application rbd 11. 创建rbd存储块 \[root@testserver01 \~\]# rbd create rbdpool01/testimg --size 5G \[root@testserver01 \~\]# rbd ls -p rbdpool01 -l NAME SIZE PARENT FMT PROT LOCK testimg 5 GiB 2 12. 添加mds服务 \[root@testserver01 \~\]# mkdir -p /var/lib/ceph/mds/ceph-testserver01 \[root@testserver01 \~\]# ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-testserver01/keyring --gen-key -n mds.testserver01 creating /var/lib/ceph/mds/ceph-testserver01/keyring \[root@testserver01 \~\]# ceph auth add mds.testserver01 osd 'allow rwx' mds 'allow \*' mon 'allow profile mds' -i /var/lib/ceph/mds/ceph-testserver01/keyring added key for mds.testserver01 \[root@testserver01 \~\]# chown -R ceph.ceph /var/lib/ceph/ \[root@testserver01 \~\]# systemctl enable ceph-mds@testserver01 Created symlink from /etc/systemd/system/ceph-mds.target.wants/[email protected] to /usr/lib/systemd/system/[email protected]. \[root@testserver01 \~\]# systemctl start ceph-mds@testserver01 13. 添加cephfs存储池 \[root@testserver01 \~\]# ceph osd pool create cephfs-metadata 16 16 pool 'cephfs-metadata' created \[root@testserver01 \~\]# ceph osd pool create cephfs-data 32 32 pool 'cephfs-data' created \[root@testserver01 \~\]# ceph fs new cephfs cephfs-metadata cephfs-data new fs with metadata pool 2 and data pool 3 \[root@testserver01 \~\]# ceph fs status cephfs - 0 clients ====== +------+--------+--------------+---------------+-------+-------+ \| Rank \| State \| MDS \| Activity \| dns \| inos \| +------+--------+--------------+---------------+-------+-------+ \| 0 \| active \| testserver01 \| Reqs: 0 /s \| 10 \| 13 \| +------+--------+--------------+---------------+-------+-------+ +-----------------+----------+-------+-------+ \| Pool \| type \| used \| avail \| +-----------------+----------+-------+-------+ \| cephfs-metadata \| metadata \| 512k \| 8697M \| \| cephfs-data \| data \| 0 \| 8697M \| +-----------------+----------+-------+-------+ +-------------+ \| Standby MDS \| +-------------+ +-------------+ MDS version: ceph version 14.2.22 (ca74598065096e6fcbd8433c8779a2be0c889351) nautilus (stable) \[root@testserver01 \~\]# 14. 部署rgw (默认7480端口) \[root@testserver01 \~\]# cp /etc/ceph/ceph.conf /tmp/ \[root@testserver01 \~\]# cd /tmp \[root@testserver01 tmp\]# ceph auth get client.bootstrap-rgw -o ceph.bootstrap-rgw.keyring exported keyring for client.bootstrap-rgw \[root@testserver01 tmp\]# ceph-deploy --overwrite-conf rgw create testserver01 \[ceph_deploy.conf\]\[DEBUG \] found configuration file at: /root/.cephdeploy.conf \[ceph_deploy.cli\]\[INFO \] Invoked (2.0.1): /usr/bin/ceph-deploy --overwrite-conf rgw create testserver01 \[ceph_deploy.cli\]\[INFO \] ceph-deploy options: \[ceph_deploy.cli\]\[INFO \] username : None \[ceph_deploy.cli\]\[INFO \] verbose : False \[ceph_deploy.cli\]\[INFO \] rgw : \[('testserver01', 'rgw.testserver01')

ceph_deploy.cli\]\[INFO \] overwrite_conf : True \[ceph_deploy.cli\]\[INFO \] subcommand : create \[ceph_deploy.cli\]\[INFO \] quiet : False \[ceph_deploy.cli\]\[INFO \] cd_conf : \ \[ceph_deploy.cli\]\[INFO \] cluster : ceph \[ceph_deploy.cli\]\[INFO \] func : \ \[ceph_deploy.cli\]\[INFO \] ceph_conf : None \[ceph_deploy.cli\]\[INFO \] default_release : False \[ceph_deploy.rgw\]\[DEBUG \] Deploying rgw, cluster ceph hosts testserver01:rgw.testserver01 \[testserver01\]\[DEBUG \] connected to host: testserver01 \[testserver01\]\[DEBUG \] detect platform information from remote host \[testserver01\]\[DEBUG \] detect machine type \[ceph_deploy.rgw\]\[INFO \] Distro info: CentOS Linux 7.9.2009 Core \[ceph_deploy.rgw\]\[DEBUG \] remote host will use systemd \[ceph_deploy.rgw\]\[DEBUG \] deploying rgw bootstrap to testserver01 \[testserver01\]\[DEBUG \] write cluster configuration to /etc/ceph/{cluster}.conf \[testserver01\]\[WARNIN\] rgw keyring does not exist yet, creating one \[testserver01\]\[DEBUG \] create a keyring file \[testserver01\]\[DEBUG \] create path recursively if it doesn't exist \[testserver01\]\[INFO \] Running command: ceph --cluster ceph --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/ceph.keyring auth get-or-create client.rgw.testserver01 osd allow rwx mon allow rw -o /var/lib/ceph/radosgw/ceph-rgw.testserver01/keyring \[testserver01\]\[INFO \] Running command: systemctl enable [email protected] \[testserver01\]\[WARNIN\] Created symlink from /etc/systemd/system/ceph-radosgw.target.wants/[email protected] to /usr/lib/systemd/system/[email protected]. \[testserver01\]\[INFO \] Running command: systemctl start [email protected] \[testserver01\]\[INFO \] Running command: systemctl enable ceph.target \[ceph_deploy.rgw\]\[INFO \] The Ceph Object Gateway (RGW) is now running on host testserver01 and default port 7480 \[root@testserver01 tmp\]# radosgw-admin user create --uid=admin --display-name="admin user" --system { "user_id": "admin", "display_name": "admin user", "email": "", "suspended": 0, "max_buckets": 1000, "subusers": \[\], "keys": \[ { "user": "admin", "access_key": "7DRHRKOVTQF0EBWHL7LZ", "secret_key": "01qnDj58DmWTjQZzCwDv9SLfC6x2bIsWgD2EgHri" } \], "swift_keys": \[\], "caps": \[\], "op_mask": "read, write, delete", "system": "true", "default_placement": "", "default_storage_class": "", "placement_tags": \[\], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": \[\], "type": "rgw", "mfa_ids": \[

}

开启并配置mgr dashboard

root@testserver01 \~\]# ceph mgr module enable prometheus (开启监控端口，9283端口，需要安装ceph-mgr-dashboard包) \[root@testserver01 \~\]# ceph config set mgr mgr/dashboard/ssl false \[root@testserver01 \~\]# ceph mgr module enable dashboard (关闭dashboard ssl，默认8080端口) \[root@testserver01 tmp\]# echo 'passwordxxx' \> password.txt \[root@testserver01 tmp\]# ceph dashboard ac-user-create admin administrator -i password.txt {"username": "admin", "lastUpdate": 1709353417, "name": null, "roles": \["administrator"\], "password": "$2b$12$FcEB.KYWVB8BCXaXxu5VJOQbUqwO0KCUOEfi2wgSnmlFPW3RqTQp2", "email": null} 监控端口： ![](https://file.jishuzhan.net/article/1764608289011666945/432629f2c8ad354ca0e7f7c8ac1454b2.webp) 访问mgr dashboard http://192.168.1.128:8080 ![](https://file.jishuzhan.net/article/1764608289011666945/37c18e5ae6ebf49955f03f328026507b.webp) 16. 配置rgw dashboard \[root@testserver01 tmp\]# radosgw-admin user info --uid=admin (查看ak和sk) \[root@testserver01 tmp\]# cat /tmp/a.txt 7DRHRKOVTQF0EBWHL7LZ \[root@testserver01 tmp\]# cat /tmp/s.txt 01qnDj58DmWTjQZzCwDv9SLfC6x2bIsWgD2EgHri \[root@testserver01 tmp\]# ceph dashboard set-rgw-api-access-key -i a.txt Option RGW_API_ACCESS_KEY updated \[root@testserver01 tmp\]# ceph dashboard set-rgw-api-secret-key -i s.txt Option RGW_API_SECRET_KEY updated 17. 查看健康状态，osd，pool，crush rule，端口等 \[root@testserver01 \~\]# ceph -s cluster: id: b080bada-7f10-11ee-8f11-666666666666 health: HEALTH_OK services: mon: 1 daemons, quorum testserver01 (age 33m) mgr: testserver01(active, since 11m) mds: cephfs:1 {0=testserver01=up:active} osd: 1 osds: 1 up (since 28m), 1 in (since 28m) rgw: 1 daemon active (testserver01) task status: data: pools: 7 pools, 144 pgs objects: 216 objects, 5.7 KiB usage: 1.0 GiB used, 9.0 GiB / 10 GiB avail pgs: 144 active+clean \[root@testserver01 \~\]# ceph osd df ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS 0 hdd 0.00999 1.00000 10 GiB 1.0 GiB 4.9 MiB 0 B 1 GiB 9.0 GiB 10.05 1.00 144 up TOTAL 10 GiB 1.0 GiB 4.9 MiB 0 B 1 GiB 9.0 GiB 10.05 MIN/MAX VAR: 1.00/1.00 STDDEV: 0 \[root@testserver01 \~\]# ceph osd pool ls detail pool 1 'rbdpool01' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 28 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd removed_snaps \[1\~3

pool 2 'cephfs-metadata' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 17 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs

pool 3 'cephfs-data' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 17 flags hashpspool stripe_width 0 application cephfs

pool 4 '.rgw.root' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 20 flags hashpspool stripe_width 0 application rgw

pool 5 'default.rgw.control' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 22 flags hashpspool stripe_width 0 application rgw

pool 6 'default.rgw.meta' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 24 flags hashpspool stripe_width 0 application rgw

pool 7 'default.rgw.log' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 26 flags hashpspool stripe_width 0 application rgw

root@testserver01 \~\]# ceph osd crush rule dump \[ { "rule_id": 0, "rule_name": "replicated_rule", "ruleset": 0, "type": 1, "min_size": 1, "max_size": 10, "steps": \[ { "op": "take", "item": -1, "item_name": "default" }, { "op": "chooseleaf_firstn", "num": 0, "type": "host" }, { "op": "emit" }

}

]

root@testserver01 \~\]# netstat -tnlp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 192.168.1.128:6809 0.0.0.0:\* LISTEN 2694/ceph-osd tcp 0 0 127.0.0.1:25 0.0.0.0:\* LISTEN 977/master tcp 0 0 192.168.1.128:6810 0.0.0.0:\* LISTEN 3037/ceph-mds tcp 0 0 192.168.1.128:6811 0.0.0.0:\* LISTEN 3037/ceph-mds tcp 0 0 192.168.1.128:3300 0.0.0.0:\* LISTEN 1944/ceph-mon tcp 0 0 192.168.1.128:6789 0.0.0.0:\* LISTEN 1944/ceph-mon tcp 0 0 192.168.1.128:6800 0.0.0.0:\* LISTEN 4480/ceph-mgr tcp 0 0 192.168.1.128:6801 0.0.0.0:\* LISTEN 4480/ceph-mgr tcp 0 0 192.168.1.128:6802 0.0.0.0:\* LISTEN 2694/ceph-osd tcp 0 0 192.168.1.128:6803 0.0.0.0:\* LISTEN 2694/ceph-osd tcp 0 0 192.168.1.128:6804 0.0.0.0:\* LISTEN 2694/ceph-osd tcp 0 0 192.168.1.128:6805 0.0.0.0:\* LISTEN 2694/ceph-osd tcp 0 0 192.168.1.128:6806 0.0.0.0:\* LISTEN 2694/ceph-osd tcp 0 0 0.0.0.0:22 0.0.0.0:\* LISTEN 873/sshd tcp 0 0 192.168.1.128:6807 0.0.0.0:\* LISTEN 2694/ceph-osd tcp 0 0 0.0.0.0:7480 0.0.0.0:\* LISTEN 3280/radosgw tcp 0 0 192.168.1.128:6808 0.0.0.0:\* LISTEN 2694/ceph-osd tcp6 0 0 ::1:25 :::\* LISTEN 977/master tcp6 0 0 :::9283 :::\* LISTEN 4480/ceph-mgr tcp6 0 0 :::8080 :::\* LISTEN 4480/ceph-mgr tcp6 0 0 :::22 :::\* LISTEN 873/sshd tcp6 0 0 :::7480 :::\* LISTEN 3280/radosgw