ceph分布式存储
安装虚拟机并配置
bash
[root@xhde ~ 11:02:13]# cat >> /etc/hosts << EOF
> 192.168.108.10 client.my.cloud client
> 192.168.108.11 ceph1.my.cloud ceph1
> 192.168.108.12 ceph2.my.cloud ceph2
> 192.168.108.13 ceph3.my.cloud ceph3
> EOF
[root@xhde ~ 11:04:28]# sed -ri 's/^SELINUX=.*/SELINUX=disabled/g'
sed: no input files
[root@xhde ~ 11:05:03]# sed -ri 's/^SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
[root@xhde ~ 11:05:23]# systemctl disable firewalld --now
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@xhde ~ 11:17:22]# cat > /etc/yum.repos.d/ceph.repo << 'EOF'
> [Ceph]
> name=Ceph
> baseurl=https://mirrors.aliyun.com/centos-vault/8-stream/storage/x86_64/ceph-pacific/
> enabled=1
> gpgcheck=0
> EOF
[root@xhde ~ 11:17:29]# dnf clean all
13 files removed
[root@xhde ~ 11:17:35]# dnf makecache
Ceph 262 kB/s | 568 kB 00:02
CentOS Stream 8 - BaseOS 972 kB/s | 10 MB 00:10
CentOS Stream 8 - AppStream 597 kB/s | 29 MB 00:48
Metadata cache created.
[root@xhde ~ 11:19:02]# dnf install -y cephadm
Last metadata expiration check: 0:01:38 ago on Wed 27 May 2026 11:18:11 AM CST.
Dependencies resolved.
====================================================================================
Package Arch Version Repo Size
====================================================================================
Installing:
cephadm noarch 2:16.2.14-2.el8s Ceph 89 k
Installing dependencies:
conmon x86_64 3:2.1.10-1.module_el8+804+f131391c appstream 58 k
container-selinux noarch 2:2.229.0-2.module_el8+847+7863d4e6 appstream 71 k
containernetworking-plugins
x86_64 1:1.4.0-2.module_el8+974+0c52b299 appstream 22 M
containers-common x86_64 2:1-81.module_el8+968+fbb249c7 appstream 151 k
criu x86_64 3.18-4.module_el8+804+f131391c appstream 564 k
dnsmasq x86_64 2.79-33.el8 appstream 325 k
fuse-overlayfs x86_64 1.13-1.module_el8+804+f131391c appstream 70 k
fuse3 x86_64 3.3.0-19.el8 baseos 55 k
fuse3-libs x86_64 3.3.0-19.el8 baseos 96 k
libnet x86_64 1.1.6-15.el8 appstream 67 k
libslirp x86_64 4.4.0-1.module_el8+804+f131391c appstream 70 k
podman-catatonit x86_64 3:4.9.4-0.1.module_el8+971+3d3df00d appstream 371 k
podman-plugins x86_64 3:4.9.4-0.1.module_el8+971+3d3df00d appstream 1.3 M
protobuf-c x86_64 1.3.0-8.el8 appstream 37 k
python3-pip noarch 9.0.3-24.el8 appstream 20 k
python3-setuptools noarch 50.3.2-1.el8s Ceph 870 k
replacing platform-python-setuptools.noarch 39.2.0-7.el8
python36 x86_64 3.6.8-39.module_el8+762+77bd8591 appstream 19 k
runc x86_64 1:1.1.12-1.module_el8+885+7da147f3 appstream 3.1 M
shadow-utils-subid x86_64 2:4.6-22.el8 baseos 113 k
slirp4netns x86_64 1.2.3-1.module_el8+951+32019cde appstream 56 k
Installing weak dependencies:
podman x86_64 3:4.9.4-0.1.module_el8+971+3d3df00d appstream 16 M
podman-gvproxy x86_64 3:4.9.4-0.1.module_el8+971+3d3df00d appstream 3.8 M
Enabling module streams:
container-tools rhel8
python36 3.6
Transaction Summary
====================================================================================
Install 23 Packages
Total download size: 49 M
Downloading Packages:
(1/23): cephadm-16.2.14-2.el8s.noarch.rpm 270 kB/s | 89 kB 00:00
(2/23): fuse3-3.3.0-19.el8.x86_64.rpm 163 kB/s | 55 kB 00:00
(3/23): shadow-utils-subid-4.6-22.el8.x86_64.rpm 1.2 MB/s | 113 kB 00:00
(4/23): fuse3-libs-3.3.0-19.el8.x86_64.rpm 696 kB/s | 96 kB 00:00
(5/23): conmon-2.1.10-1.module_el8+804+f131391c.x86 529 kB/s | 58 kB 00:00
(6/23): container-selinux-2.229.0-2.module_el8+847+ 562 kB/s | 71 kB 00:00
(7/23): containers-common-1-81.module_el8+968+fbb24 651 kB/s | 151 kB 00:00
(8/23): python3-setuptools-50.3.2-1.el8s.noarch.rpm 424 kB/s | 870 kB 00:02
(9/23): criu-3.18-4.module_el8+804+f131391c.x86_64. 445 kB/s | 564 kB 00:01
(10/23): fuse-overlayfs-1.13-1.module_el8+804+f1313 369 kB/s | 70 kB 00:00
(11/23): libnet-1.1.6-15.el8.x86_64.rpm 301 kB/s | 67 kB 00:00
(12/23): libslirp-4.4.0-1.module_el8+804+f131391c.x 1.0 MB/s | 70 kB 00:00
(13/23): dnsmasq-2.79-33.el8.x86_64.rpm 524 kB/s | 325 kB 00:00
(14/23): podman-catatonit-4.9.4-0.1.module_el8+971+ 553 kB/s | 371 kB 00:00
(15/23): podman-gvproxy-4.9.4-0.1.module_el8+971+3d 478 kB/s | 3.8 MB 00:08
(16/23): podman-plugins-4.9.4-0.1.module_el8+971+3d 601 kB/s | 1.3 MB 00:02
(17/23): protobuf-c-1.3.0-8.el8.x86_64.rpm 698 kB/s | 37 kB 00:00
(18/23): python3-pip-9.0.3-24.el8.noarch.rpm 365 kB/s | 20 kB 00:00
(19/23): python36-3.6.8-39.module_el8+762+77bd8591. 347 kB/s | 19 kB 00:00
(20/23): runc-1.1.12-1.module_el8+885+7da147f3.x86_ 506 kB/s | 3.1 MB 00:06
(21/23): slirp4netns-1.2.3-1.module_el8+951+32019cd 351 kB/s | 56 kB 00:00
(22/23): podman-4.9.4-0.1.module_el8+971+3d3df00d.x 495 kB/s | 16 MB 00:33
(23/23): containernetworking-plugins-1.4.0-2.module 496 kB/s | 22 MB 00:45
------------------------------------------------------------------------------------
Total 1.1 MB/s | 49 MB 00:45
Running transaction check
Transaction check succeeded.
Running transaction test
Transaction test succeeded.
Running transaction
Preparing : 1/1
Running scriptlet: container-selinux-2:2.229.0-2.module_el8+847+7863d4e6. 1/24
Installing : container-selinux-2:2.229.0-2.module_el8+847+7863d4e6. 1/24
Running scriptlet: container-selinux-2:2.229.0-2.module_el8+847+7863d4e6. 1/24
Installing : fuse3-libs-3.3.0-19.el8.x86_64 2/24
Running scriptlet: fuse3-libs-3.3.0-19.el8.x86_64 2/24
Installing : fuse3-3.3.0-19.el8.x86_64 3/24
Installing : fuse-overlayfs-1.13-1.module_el8+804+f131391c.x86_64 4/24
Running scriptlet: fuse-overlayfs-1.13-1.module_el8+804+f131391c.x86_64 4/24
Installing : protobuf-c-1.3.0-8.el8.x86_64 5/24
Installing : podman-gvproxy-3:4.9.4-0.1.module_el8+971+3d3df00d.x86 6/24
Installing : libslirp-4.4.0-1.module_el8+804+f131391c.x86_64 7/24
Installing : slirp4netns-1.2.3-1.module_el8+951+32019cde.x86_64 8/24
Installing : libnet-1.1.6-15.el8.x86_64 9/24
Running scriptlet: libnet-1.1.6-15.el8.x86_64 9/24
Installing : criu-3.18-4.module_el8+804+f131391c.x86_64 10/24
Installing : runc-1:1.1.12-1.module_el8+885+7da147f3.x86_64 11/24
Installing : containers-common-2:1-81.module_el8+968+fbb249c7.x86_6 12/24
Running scriptlet: dnsmasq-2.79-33.el8.x86_64 13/24
Installing : dnsmasq-2.79-33.el8.x86_64 13/24
Running scriptlet: dnsmasq-2.79-33.el8.x86_64 13/24
Installing : podman-plugins-3:4.9.4-0.1.module_el8+971+3d3df00d.x86 14/24
Installing : containernetworking-plugins-1:1.4.0-2.module_el8+974+0 15/24
Installing : conmon-3:2.1.10-1.module_el8+804+f131391c.x86_64 16/24
Installing : shadow-utils-subid-2:4.6-22.el8.x86_64 17/24
Installing : podman-catatonit-3:4.9.4-0.1.module_el8+971+3d3df00d.x 18/24
Installing : podman-3:4.9.4-0.1.module_el8+971+3d3df00d.x86_64 19/24
Installing : python3-setuptools-50.3.2-1.el8s.noarch 20/24
Installing : python3-pip-9.0.3-24.el8.noarch 21/24
Installing : python36-3.6.8-39.module_el8+762+77bd8591.x86_64 22/24
Running scriptlet: python36-3.6.8-39.module_el8+762+77bd8591.x86_64 22/24
Running scriptlet: cephadm-2:16.2.14-2.el8s.noarch 23/24
Installing : cephadm-2:16.2.14-2.el8s.noarch 23/24
Obsoleting : platform-python-setuptools-39.2.0-7.el8.noarch 24/24
Running scriptlet: container-selinux-2:2.229.0-2.module_el8+847+7863d4e6. 24/24
Running scriptlet: platform-python-setuptools-39.2.0-7.el8.noarch 24/24
Verifying : cephadm-2:16.2.14-2.el8s.noarch 1/24
Verifying : python3-setuptools-50.3.2-1.el8s.noarch 2/24
Verifying : platform-python-setuptools-39.2.0-7.el8.noarch 3/24
Verifying : fuse3-3.3.0-19.el8.x86_64 4/24
Verifying : fuse3-libs-3.3.0-19.el8.x86_64 5/24
Verifying : shadow-utils-subid-2:4.6-22.el8.x86_64 6/24
Verifying : conmon-3:2.1.10-1.module_el8+804+f131391c.x86_64 7/24
Verifying : container-selinux-2:2.229.0-2.module_el8+847+7863d4e6. 8/24
Verifying : containernetworking-plugins-1:1.4.0-2.module_el8+974+0 9/24
Verifying : containers-common-2:1-81.module_el8+968+fbb249c7.x86_6 10/24
Verifying : criu-3.18-4.module_el8+804+f131391c.x86_64 11/24
Verifying : dnsmasq-2.79-33.el8.x86_64 12/24
Verifying : fuse-overlayfs-1.13-1.module_el8+804+f131391c.x86_64 13/24
Verifying : libnet-1.1.6-15.el8.x86_64 14/24
Verifying : libslirp-4.4.0-1.module_el8+804+f131391c.x86_64 15/24
Verifying : podman-3:4.9.4-0.1.module_el8+971+3d3df00d.x86_64 16/24
Verifying : podman-catatonit-3:4.9.4-0.1.module_el8+971+3d3df00d.x 17/24
Verifying : podman-gvproxy-3:4.9.4-0.1.module_el8+971+3d3df00d.x86 18/24
Verifying : podman-plugins-3:4.9.4-0.1.module_el8+971+3d3df00d.x86 19/24
Verifying : protobuf-c-1.3.0-8.el8.x86_64 20/24
Verifying : python3-pip-9.0.3-24.el8.noarch 21/24
Verifying : python36-3.6.8-39.module_el8+762+77bd8591.x86_64 22/24
Verifying : runc-1:1.1.12-1.module_el8+885+7da147f3.x86_64 23/24
Verifying : slirp4netns-1.2.3-1.module_el8+951+32019cde.x86_64 24/24
Installed:
cephadm-2:16.2.14-2.el8s.noarch
conmon-3:2.1.10-1.module_el8+804+f131391c.x86_64
container-selinux-2:2.229.0-2.module_el8+847+7863d4e6.noarch
containernetworking-plugins-1:1.4.0-2.module_el8+974+0c52b299.x86_64
containers-common-2:1-81.module_el8+968+fbb249c7.x86_64
criu-3.18-4.module_el8+804+f131391c.x86_64
dnsmasq-2.79-33.el8.x86_64
fuse-overlayfs-1.13-1.module_el8+804+f131391c.x86_64
fuse3-3.3.0-19.el8.x86_64
fuse3-libs-3.3.0-19.el8.x86_64
libnet-1.1.6-15.el8.x86_64
libslirp-4.4.0-1.module_el8+804+f131391c.x86_64
podman-3:4.9.4-0.1.module_el8+971+3d3df00d.x86_64
podman-catatonit-3:4.9.4-0.1.module_el8+971+3d3df00d.x86_64
podman-gvproxy-3:4.9.4-0.1.module_el8+971+3d3df00d.x86_64
podman-plugins-3:4.9.4-0.1.module_el8+971+3d3df00d.x86_64
protobuf-c-1.3.0-8.el8.x86_64
python3-pip-9.0.3-24.el8.noarch
python3-setuptools-50.3.2-1.el8s.noarch
python36-3.6.8-39.module_el8+762+77bd8591.x86_64
runc-1:1.1.12-1.module_el8+885+7da147f3.x86_64
shadow-utils-subid-2:4.6-22.el8.x86_64
slirp4netns-1.2.3-1.module_el8+951+32019cde.x86_64
Complete!
[root@xhde ~ 11:22:14]# cd /etc/yum.repos.d
[root@xhde yum.repos.d 11:24:18]# ls
ceph.repo os.repo
[root@xhde yum.repos.d 11:24:20]# cd ~
[root@xhde ~ 11:24:46]# rpm -q podman
podman-4.9.4-0.1.module_el8+971+3d3df00d.x86_64
[root@xhde ~ 11:24:46]#
podman pull quay.io/ceph/ceph:v16
podman pull quay.io/ceph/ceph-grafana:8.3.5
podman pull quay.io/prometheus/node-exporter:v1.3.1
podman pull quay.io/prometheus/alertmanager:v0.23.0
podman pull quay.io/prometheus/prometheus:v2.33.4
bash
#查看脚本
[root@xhde ~ 11:34:49]# cat > /usr/local/bin/sethost <<'EOF'
> #/bin/bash
> hostnamectl set-hostname ceph$1.my.cloud
> nmcli connection modify ens160 ipv4.method manual ipv4.addresses 192.168.108.1$1/24 ipv4.gateway 192.168.108.2 ipv4.dns 192.168.108.2
> init 0
> EOF
[root@xhde ~ 11:40:56]# chmod +x /usr/local/bin/sethost
[root@xhde ~ 11:41:03]# vim /usr/local/bin/sethost
bash
#ceph1配置
[root@ceph1 ~ 11:59:13]# cephadm bootstrap --mon-ip 192.168.108.11 --allow-fqdn-hostname --initial-dashboard-user admin --initial-dashboard-password my@123 --dashboard-password-noupdate
Creating directory /etc/ceph for ceph.conf
Verifying podman|docker is present...
Verifying lvm2 is present...
Verifying time synchronization is in place...
Unit chronyd.service is enabled and running
Repeating the final host check...
podman (/usr/bin/podman) version 4.9.4 is present
systemctl is present
lvcreate is present
Unit chronyd.service is enabled and running
Host looks OK
Cluster fsid: 96b28050-5980-11f1-8411-000c2993506d
Verifying IP 192.168.108.11 port 3300 ...
Verifying IP 192.168.108.11 port 6789 ...
Mon IP `192.168.108.11` is in CIDR network `192.168.108.0/24`
Mon IP `192.168.108.11` is in CIDR network `192.168.108.0/24`
Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network
Pulling container image quay.io/ceph/ceph:v16...
Ceph version: ceph version 16.2.15 (618f440892089921c3e944a991122ddc44e60516) pacific (stable)
Extracting ceph user uid/gid from container image...
Creating initial keys...
Creating initial monmap...
Creating mon...
Waiting for mon to start...
Waiting for mon...
mon is available
Assimilating anything we can from ceph.conf...
Generating new minimal ceph.conf...
Restarting the monitor...
Setting public_network to 192.168.108.0/24 in mon config section
Wrote config to /etc/ceph/ceph.conf
Wrote keyring to /etc/ceph/ceph.client.admin.keyring
Creating mgr...
Verifying port 9283 ...
Waiting for mgr to start...
Waiting for mgr...
mgr not available, waiting (1/15)...
mgr not available, waiting (2/15)...
mgr not available, waiting (3/15)...
mgr not available, waiting (4/15)...
mgr is available
Enabling cephadm module...
Waiting for the mgr to restart...
Waiting for mgr epoch 5...
mgr epoch 5 is available
Setting orchestrator backend to cephadm...
Generating ssh key...
Wrote public SSH key to /etc/ceph/ceph.pub
Adding key to root@localhost authorized_keys...
Adding host ceph1.my.cloud...
Deploying mon service with default placement...
Deploying mgr service with default placement...
Deploying crash service with default placement...
Deploying prometheus service with default placement...
Deploying grafana service with default placement...
Deploying node-exporter service with default placement...
Deploying alertmanager service with default placement...
Enabling the dashboard module...
Waiting for the mgr to restart...
Waiting for mgr epoch 9...
mgr epoch 9 is available
Generating a dashboard self-signed certificate...
Creating initial admin user...
Fetching dashboard port number...
Ceph Dashboard is now available at:
URL: https://ceph1.my.cloud:8443/
User: admin
Password: my@123
Enabling client.admin keyring and conf on hosts with "admin" label
Enabling autotune for osd_memory_target
You can access the Ceph CLI as following in case of multi-cluster or non-default config:
sudo /usr/sbin/cephadm shell --fsid 96b28050-5980-11f1-8411-000c2993506d -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring
Or, if you are only running a single cluster on this host:
sudo /usr/sbin/cephadm shell
Please consider enabling telemetry to help improve Ceph:
ceph telemetry on
For more information see:
https://docs.ceph.com/en/pacific/mgr/telemetry/
Bootstrap complete.
bash
#ceph集群部署
[root@ceph1 ~ 12:01:42]# dnf install -y ceph-common
Last metadata expiration check: 1:13:18 ago on Wed 27 May 2026 12:52:48 PM CST.
Dependencies resolved.
====================================================================================
Package Arch Version Repository Size
====================================================================================
Installing:
ceph-common x86_64 2:16.2.15-1.el8s Ceph 24 M
Installing dependencies:
gperftools-libs x86_64 2.9.1-1.el8s Ceph 311 k
leveldb x86_64 1.20-1.el8s Ceph 165 k
libbabeltrace x86_64 1.5.4-4.el8 baseos 200 k
libcephfs2 x86_64 2:16.2.15-1.el8s Ceph 860 k
libicu x86_64 60.3-2.el8_1 baseos 8.8 M
liboath x86_64 2.6.2-4.el8s Ceph 59 k
librabbitmq x86_64 0.9.0-5.el8 baseos 47 k
librados2 x86_64 2:16.2.15-1.el8s Ceph 3.8 M
libradosstriper1 x86_64 2:16.2.15-1.el8s Ceph 528 k
librbd1 x86_64 2:16.2.15-1.el8s Ceph 4.2 M
librdkafka x86_64 1.6.1-1.el8 appstream 677 k
librdmacm x86_64 48.0-1.el8 baseos 79 k
librgw2 x86_64 2:16.2.15-1.el8s Ceph 3.8 M
libunwind x86_64 1.4.0-5.el8s Ceph 75 k
lttng-ust x86_64 2.8.1-11.el8 appstream 259 k
python3-ceph-argparse x86_64 2:16.2.15-1.el8s Ceph 71 k
python3-ceph-common x86_64 2:16.2.15-1.el8s Ceph 125 k
python3-cephfs x86_64 2:16.2.15-1.el8s Ceph 241 k
python3-prettytable noarch 0.7.2-14.el8 appstream 44 k
python3-pyyaml x86_64 3.12-12.el8 baseos 193 k
python3-rados x86_64 2:16.2.15-1.el8s Ceph 415 k
python3-rbd x86_64 2:16.2.15-1.el8s Ceph 395 k
python3-rgw x86_64 2:16.2.15-1.el8s Ceph 141 k
userspace-rcu x86_64 0.10.1-4.el8 baseos 101 k
Transaction Summary
====================================================================================
Install 25 Packages
Total download size: 50 M
Installed size: 166 M
Downloading Packages:
(1/25): leveldb-1.20-1.el8s.x86_64.rpm 1.2 MB/s | 165 kB 00:00
(2/25): gperftools-libs-2.9.1-1.el8s.x86_64.rpm 1.0 MB/s | 311 kB 00:00
(3/25): liboath-2.6.2-4.el8s.x86_64.rpm 2.5 MB/s | 59 kB 00:00
(4/25): libcephfs2-16.2.15-1.el8s.x86_64.rpm 1.0 MB/s | 860 kB 00:00
(5/25): libradosstriper1-16.2.15-1.el8s.x86_64.rpm 1.0 MB/s | 528 kB 00:00
(6/25): librados2-16.2.15-1.el8s.x86_64.rpm 896 kB/s | 3.8 MB 00:04
(7/25): librbd1-16.2.15-1.el8s.x86_64.rpm 925 kB/s | 4.2 MB 00:04
(8/25): libunwind-1.4.0-5.el8s.x86_64.rpm 2.7 MB/s | 75 kB 00:00
(9/25): python3-ceph-argparse-16.2.15-1.el8s.x86_64 4.7 MB/s | 71 kB 00:00
(10/25): python3-ceph-common-16.2.15-1.el8s.x86_64. 6.2 MB/s | 125 kB 00:00
(11/25): python3-cephfs-16.2.15-1.el8s.x86_64.rpm 7.2 MB/s | 241 kB 00:00
(12/25): python3-rados-16.2.15-1.el8s.x86_64.rpm 971 kB/s | 415 kB 00:00
(13/25): python3-rbd-16.2.15-1.el8s.x86_64.rpm 802 kB/s | 395 kB 00:00
(14/25): python3-rgw-16.2.15-1.el8s.x86_64.rpm 3.2 MB/s | 141 kB 00:00
(15/25): libbabeltrace-1.5.4-4.el8.x86_64.rpm 4.6 MB/s | 200 kB 00:00
(16/25): librgw2-16.2.15-1.el8s.x86_64.rpm 855 kB/s | 3.8 MB 00:04
(17/25): librabbitmq-0.9.0-5.el8.x86_64.rpm 865 kB/s | 47 kB 00:00
(18/25): librdmacm-48.0-1.el8.x86_64.rpm 2.3 MB/s | 79 kB 00:00
(19/25): python3-pyyaml-3.12-12.el8.x86_64.rpm 3.8 MB/s | 193 kB 00:00
(20/25): userspace-rcu-0.10.1-4.el8.x86_64.rpm 2.2 MB/s | 101 kB 00:00
(21/25): librdkafka-1.6.1-1.el8.x86_64.rpm 880 kB/s | 677 kB 00:00
(22/25): lttng-ust-2.8.1-11.el8.x86_64.rpm 4.9 MB/s | 259 kB 00:00
(23/25): python3-prettytable-0.7.2-14.el8.noarch.rp 2.5 MB/s | 44 kB 00:00
(24/25): libicu-60.3-2.el8_1.x86_64.rpm 879 kB/s | 8.8 MB 00:10
(25/25): ceph-common-16.2.15-1.el8s.x86_64.rpm 914 kB/s | 24 MB 00:27
------------------------------------------------------------------------------------
Total 1.8 MB/s | 50 MB 00:27
Running transaction check
Transaction check succeeded.
Running transaction test
Transaction test succeeded.
Running transaction
Preparing : 1/1
Installing : librdmacm-48.0-1.el8.x86_64 1/25
Installing : librdkafka-1.6.1-1.el8.x86_64 2/25
Installing : librabbitmq-0.9.0-5.el8.x86_64 3/25
Installing : libicu-60.3-2.el8_1.x86_64 4/25
Running scriptlet: libicu-60.3-2.el8_1.x86_64 4/25
Installing : python3-ceph-argparse-2:16.2.15-1.el8s.x86_64 5/25
Installing : liboath-2.6.2-4.el8s.x86_64 6/25
Installing : python3-prettytable-0.7.2-14.el8.noarch 7/25
Installing : userspace-rcu-0.10.1-4.el8.x86_64 8/25
Running scriptlet: userspace-rcu-0.10.1-4.el8.x86_64 8/25
Installing : lttng-ust-2.8.1-11.el8.x86_64 9/25
Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 9/25
Installing : librados2-2:16.2.15-1.el8s.x86_64 10/25
Running scriptlet: librados2-2:16.2.15-1.el8s.x86_64 10/25
Installing : python3-rados-2:16.2.15-1.el8s.x86_64 11/25
Installing : libcephfs2-2:16.2.15-1.el8s.x86_64 12/25
Running scriptlet: libcephfs2-2:16.2.15-1.el8s.x86_64 12/25
Installing : librbd1-2:16.2.15-1.el8s.x86_64 13/25
Running scriptlet: librbd1-2:16.2.15-1.el8s.x86_64 13/25
Installing : python3-rbd-2:16.2.15-1.el8s.x86_64 14/25
Installing : python3-cephfs-2:16.2.15-1.el8s.x86_64 15/25
Installing : libradosstriper1-2:16.2.15-1.el8s.x86_64 16/25
Running scriptlet: libradosstriper1-2:16.2.15-1.el8s.x86_64 16/25
Installing : librgw2-2:16.2.15-1.el8s.x86_64 17/25
Running scriptlet: librgw2-2:16.2.15-1.el8s.x86_64 17/25
Installing : python3-rgw-2:16.2.15-1.el8s.x86_64 18/25
Installing : python3-pyyaml-3.12-12.el8.x86_64 19/25
Installing : python3-ceph-common-2:16.2.15-1.el8s.x86_64 20/25
Installing : libbabeltrace-1.5.4-4.el8.x86_64 21/25
Running scriptlet: libbabeltrace-1.5.4-4.el8.x86_64 21/25
Installing : libunwind-1.4.0-5.el8s.x86_64 22/25
Installing : gperftools-libs-2.9.1-1.el8s.x86_64 23/25
Installing : leveldb-1.20-1.el8s.x86_64 24/25
Running scriptlet: leveldb-1.20-1.el8s.x86_64 24/25
Running scriptlet: ceph-common-2:16.2.15-1.el8s.x86_64 25/25
Installing : ceph-common-2:16.2.15-1.el8s.x86_64 25/25
Running scriptlet: ceph-common-2:16.2.15-1.el8s.x86_64 25/25
Verifying : ceph-common-2:16.2.15-1.el8s.x86_64 1/25
Verifying : gperftools-libs-2.9.1-1.el8s.x86_64 2/25
Verifying : leveldb-1.20-1.el8s.x86_64 3/25
Verifying : libcephfs2-2:16.2.15-1.el8s.x86_64 4/25
Verifying : liboath-2.6.2-4.el8s.x86_64 5/25
Verifying : librados2-2:16.2.15-1.el8s.x86_64 6/25
Verifying : libradosstriper1-2:16.2.15-1.el8s.x86_64 7/25
Verifying : librbd1-2:16.2.15-1.el8s.x86_64 8/25
Verifying : librgw2-2:16.2.15-1.el8s.x86_64 9/25
Verifying : libunwind-1.4.0-5.el8s.x86_64 10/25
Verifying : python3-ceph-argparse-2:16.2.15-1.el8s.x86_64 11/25
Verifying : python3-ceph-common-2:16.2.15-1.el8s.x86_64 12/25
Verifying : python3-cephfs-2:16.2.15-1.el8s.x86_64 13/25
Verifying : python3-rados-2:16.2.15-1.el8s.x86_64 14/25
Verifying : python3-rbd-2:16.2.15-1.el8s.x86_64 15/25
Verifying : python3-rgw-2:16.2.15-1.el8s.x86_64 16/25
Verifying : libbabeltrace-1.5.4-4.el8.x86_64 17/25
Verifying : libicu-60.3-2.el8_1.x86_64 18/25
Verifying : librabbitmq-0.9.0-5.el8.x86_64 19/25
Verifying : librdmacm-48.0-1.el8.x86_64 20/25
Verifying : python3-pyyaml-3.12-12.el8.x86_64 21/25
Verifying : userspace-rcu-0.10.1-4.el8.x86_64 22/25
Verifying : librdkafka-1.6.1-1.el8.x86_64 23/25
Verifying : lttng-ust-2.8.1-11.el8.x86_64 24/25
Verifying : python3-prettytable-0.7.2-14.el8.noarch 25/25
Installed:
ceph-common-2:16.2.15-1.el8s.x86_64
gperftools-libs-2.9.1-1.el8s.x86_64
leveldb-1.20-1.el8s.x86_64
libbabeltrace-1.5.4-4.el8.x86_64
libcephfs2-2:16.2.15-1.el8s.x86_64
libicu-60.3-2.el8_1.x86_64
liboath-2.6.2-4.el8s.x86_64
librabbitmq-0.9.0-5.el8.x86_64
librados2-2:16.2.15-1.el8s.x86_64
libradosstriper1-2:16.2.15-1.el8s.x86_64
librbd1-2:16.2.15-1.el8s.x86_64
librdkafka-1.6.1-1.el8.x86_64
librdmacm-48.0-1.el8.x86_64
librgw2-2:16.2.15-1.el8s.x86_64
libunwind-1.4.0-5.el8s.x86_64
lttng-ust-2.8.1-11.el8.x86_64
python3-ceph-argparse-2:16.2.15-1.el8s.x86_64
python3-ceph-common-2:16.2.15-1.el8s.x86_64
python3-cephfs-2:16.2.15-1.el8s.x86_64
python3-prettytable-0.7.2-14.el8.noarch
python3-pyyaml-3.12-12.el8.x86_64
python3-rados-2:16.2.15-1.el8s.x86_64
python3-rbd-2:16.2.15-1.el8s.x86_64
python3-rgw-2:16.2.15-1.el8s.x86_64
userspace-rcu-0.10.1-4.el8.x86_64
Complete!
[root@ceph1 ~ 14:06:46]# ceph cephadm get-pub-key > ~/ceph.pub
[root@ceph1 ~ 14:07:06]# ssh-copy-id -f -i ~/ceph.pub root@ceph2.my.cloud
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/ceph.pub"
ssh: connect to host ceph2.my.cloud port 22: No route to host
[root@ceph1 ~ 14:07:44]# ssh-copy-id -f -i ~/ceph.pub root@ceph2.my.cloud
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/ceph.pub"
The authenticity of host 'ceph2.my.cloud (192.168.108.12)' can't be established.
ECDSA key fingerprint is SHA256:5nUPAe0IfgEk1mP8rsIRsrWNDyd8rjXi6L1OJrSdbRE.
Are you sure you want to continue connecting (yes/no/[fingerprint])? y
Please type 'yes', 'no' or the fingerprint: yes
root@ceph2.my.cloud's password:
Number of key(s) added: 1
Now try logging into the machine, with: "ssh 'root@ceph2.my.cloud'"
and check to make sure that only the key(s) you wanted were added.
[root@ceph1 ~ 14:08:31]# ssh-copy-id -f -i ~/ceph.pub root@ceph3.my.cloud
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/ceph.pub"
The authenticity of host 'ceph3.my.cloud (192.168.108.13)' can't be established.
ECDSA key fingerprint is SHA256:5nUPAe0IfgEk1mP8rsIRsrWNDyd8rjXi6L1OJrSdbRE.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
root@ceph3.my.cloud's password:
Number of key(s) added: 1
Now try logging into the machine, with: "ssh 'root@ceph3.my.cloud'"
and check to make sure that only the key(s) you wanted were added.
[root@ceph1 ~ 14:09:21]# ceph orch host add ceph2.my.cloud
Added host 'ceph2.my.cloud' with addr '192.168.108.12'
[root@ceph1 ~ 14:09:45]# ceph orch host add ceph3.my.cloud
Added host 'ceph3.my.cloud' with addr '192.168.108.13'
[root@ceph1 ~ 14:09:53]# ceph orch host ls
HOST ADDR LABELS STATUS
ceph1.my.cloud 192.168.108.11 _admin
ceph2.my.cloud 192.168.108.12
ceph3.my.cloud 192.168.108.13
3 hosts in cluster
[root@ceph1 ~ 14:10:03]# ceph orch ls
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
alertmanager ?:9093,9094 1/1 6m ago 2h count:1
crash 3/3 6m ago 2h *
grafana ?:3000 1/1 6m ago 2h count:1
mgr 2/2 6m ago 2h count:2
mon 3/5 6m ago 2h count:5
node-exporter ?:9100 3/3 6m ago 2h *
prometheus ?:9095 1/1 6m ago 2h count:1
[root@ceph1 ~ 14:10:20]# ceph orch apply mon --unmanaged=true
Scheduled mon update...
[root@ceph1 ~ 14:10:32]# ceph orch apply mgr --unmanaged=true
Scheduled mgr update...
[root@ceph1 ~ 14:10:44]# ceph orch ls
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
alertmanager ?:9093,9094 1/1 15s ago 2h count:1
crash 3/3 16s ago 2h *
grafana ?:3000 1/1 15s ago 2h count:1
mgr 2/2 16s ago 5s <unmanaged>
mon 3/5 16s ago 17s <unmanaged>
node-exporter ?:9100 3/3 16s ago 2h *
prometheus ?:9095 1/1 15s ago 2h count:1
[root@ceph1 ~ 14:10:50]# ceph orch host label add ceph2.my.cloud _admin
Added label _admin to host ceph2.my.cloud
[root@ceph1 ~ 14:11:25]# ceph orch host label add ceph3.my.cloud _admin
Added label _admin to host ceph3.my.cloud
[root@ceph1 ~ 14:11:31]# ceph orch host ls
HOST ADDR LABELS STATUS
ceph1.my.cloud 192.168.108.11 _admin
ceph2.my.cloud 192.168.108.12 _admin
ceph3.my.cloud 192.168.108.13 _admin
3 hosts in cluster
[root@ceph1 ~ 14:11:42]# ceph orch apply mon --placement="label:_admin"
Scheduled mon update...
[root@ceph1 ~ 14:11:56]# ceph orch apply mgr --placement="label:_admin"
Scheduled mgr update...
[root@ceph1 ~ 14:12:09]# ceph orch ls | egrep 'mon|mgr'
mgr 3/3 103s ago 7s label:_admin
mon 3/3 103s ago 20s label:_admin
[root@ceph1 ~ 14:12:16]# ceph orch ps | egrep 'mon|mgr'
mgr.ceph1.my.cloud.newsth ceph1.my.cloud *:9283 running (2h) 2s ago 2h 448M - 16.2.15 3c4eff6082ae 43e2adc0f43b
mgr.ceph2.aessly ceph2.my.cloud *:8443,9283 running (2m) 108s ago 2m 384M - 16.2.15 3c4eff6082ae ee614cdf998b
mgr.ceph3.ltrhra ceph3.my.cloud *:8443,9283 running (11s) 10s ago 11s 37.6M - 16.2.15 3c4eff6082ae 19aeb783c8ff
mon.ceph1.my.cloud ceph1.my.cloud running (2h) 2s ago 2h 304M 2048M 16.2.15 3c4eff6082ae 4662203936e6
mon.ceph2 ceph2.my.cloud running (2m) 108s ago 2m 83.8M 2048M 16.2.15 3c4eff6082ae 29dca1b972e5
mon.ceph3 ceph3.my.cloud running (2m) 10s ago 2m 81.5M 2048M 16.2.15 3c4eff6082ae 6773ff523077
[root@ceph1 ~ 14:12:22]# ceph orch apply osd --all-available-devices
Scheduled osd.all-available-devices update...
[root@ceph1 ~ 14:12:38]# ceph orch ls
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
alertmanager ?:9093,9094 1/1 6s ago 2h count:1
crash 3/3 2m ago 2h *
grafana ?:3000 1/1 6s ago 2h count:1
mgr 3/3 2m ago 37s label:_admin
mon 3/3 2m ago 50s label:_admin
node-exporter ?:9100 3/3 2m ago 2h *
osd.all-available-devices 0 - 8s *
prometheus ?:9095 1/1 6s ago 2h count:1
[root@ceph1 ~ 14:12:47]# ceph -s
cluster:
id: 96b28050-5980-11f1-8411-000c2993506d
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph1.my.cloud,ceph2,ceph3 (age 2m)
mgr: ceph1.my.cloud.newsth(active, since 2h), standbys: ceph2.aessly, ceph3.ltrhra
osd: 9 osds: 0 up, 9 in (since 5s)
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[root@ceph1 ~ 14:12:59]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0 root default
0 0 osd.0 down 1.00000 1.00000
1 0 osd.1 down 1.00000 1.00000
2 0 osd.2 down 1.00000 1.00000
3 0 osd.3 down 1.00000 1.00000
4 0 osd.4 down 1.00000 1.00000
5 0 osd.5 down 1.00000 1.00000
6 0 osd.6 down 1.00000 1.00000
7 0 osd.7 down 1.00000 1.00000
8 0 osd.8 down 1.00000 1.00000
[root@ceph1 ~ 14:13:08]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.17537 root default
-7 0.05846 host ceph1
2 ssd 0.01949 osd.2 up 1.00000 1.00000
5 ssd 0.01949 osd.5 up 1.00000 1.00000
8 ssd 0.01949 osd.8 up 1.00000 1.00000
-5 0.05846 host ceph2
0 ssd 0.01949 osd.0 up 1.00000 1.00000
3 ssd 0.01949 osd.3 up 1.00000 1.00000
6 ssd 0.01949 osd.6 up 1.00000 1.00000
-3 0.05846 host ceph3
1 ssd 0.01949 osd.1 up 1.00000 1.00000
4 ssd 0.01949 osd.4 up 1.00000 1.00000
7 ssd 0.01949 osd.7 up 1.00000 1.00000
[root@ceph1 ~ 14:14:47]# ceph orch ps
NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID
alertmanager.ceph1 ceph1.my.cloud *:9093,9094 running (3m) 43s ago 2h 22.4M - 0.23.0 ba2b418f427c 47b7c1474c22
crash.ceph1 ceph1.my.cloud running (2h) 43s ago 2h 6647k - 16.2.15 3c4eff6082ae 84d0570754d7
crash.ceph2 ceph2.my.cloud running (5m) 45s ago 5m 6643k - 16.2.15 3c4eff6082ae c3b5040a5452
crash.ceph3 ceph3.my.cloud running (5m) 45s ago 5m 6639k - 16.2.15 3c4eff6082ae a613c9b30e6b
grafana.ceph1 ceph1.my.cloud *:3000 running (2h) 43s ago 2h 53.3M - 8.3.5 dad864ee21e9 ee7cc1b69828
mgr.ceph1.my.cloud.newsth ceph1.my.cloud *:9283 running (2h) 43s ago 2h 459M - 16.2.15 3c4eff6082ae 43e2adc0f43b
mgr.ceph2.aessly ceph2.my.cloud *:8443,9283 running (5m) 45s ago 5m 387M - 16.2.15 3c4eff6082ae ee614cdf998b
mgr.ceph3.ltrhra ceph3.my.cloud *:8443,9283 running (3m) 45s ago 3m 387M - 16.2.15 3c4eff6082ae 19aeb783c8ff
mon.ceph1.my.cloud ceph1.my.cloud running (2h) 43s ago 2h 316M 2048M 16.2.15 3c4eff6082ae 4662203936e6
mon.ceph2 ceph2.my.cloud running (5m) 45s ago 5m 63.5M 2048M 16.2.15 3c4eff6082ae 29dca1b972e5
mon.ceph3 ceph3.my.cloud running (5m) 45s ago 5m 40.2M 2048M 16.2.15 3c4eff6082ae 6773ff523077
node-exporter.ceph1 ceph1.my.cloud *:9100 running (2h) 43s ago 2h 21.6M - 1.3.1 1dbe0e931976 a629054d4788
node-exporter.ceph2 ceph2.my.cloud *:9100 running (5m) 45s ago 5m 18.9M - 1.3.1 1dbe0e931976 5f652055674c
node-exporter.ceph3 ceph3.my.cloud *:9100 running (5m) 45s ago 5m 18.6M - 1.3.1 1dbe0e931976 d43158bcea4d
osd.0 ceph2.my.cloud running (2m) 45s ago 2m 26.1M 4096M 16.2.15 3c4eff6082ae c3e076657690
osd.1 ceph3.my.cloud running (2m) 45s ago 2m 24.9M 4096M 16.2.15 3c4eff6082ae 55974f785a3c
osd.2 ceph1.my.cloud running (2m) 43s ago 2m 26.0M 4096M 16.2.15 3c4eff6082ae fd69df193b31
osd.3 ceph2.my.cloud running (2m) 45s ago 2m 26.8M 4096M 16.2.15 3c4eff6082ae 6ae7a61507a7
osd.4 ceph3.my.cloud running (2m) 45s ago 2m 24.6M 4096M 16.2.15 3c4eff6082ae d0b2c9d7a80c
osd.5 ceph1.my.cloud running (2m) 43s ago 2m 25.1M 4096M 16.2.15 3c4eff6082ae 15ae11ee65bf
osd.6 ceph2.my.cloud running (2m) 45s ago 2m 24.9M 4096M 16.2.15 3c4eff6082ae a42dbb5943c3
osd.7 ceph3.my.cloud running (2m) 45s ago 2m 24.5M 4096M 16.2.15 3c4eff6082ae d75a4c1eeb69
osd.8 ceph1.my.cloud running (2m) 43s ago 2m 24.7M 4096M 16.2.15 3c4eff6082ae 190f5dd8b3b8
prometheus.ceph1 ceph1.my.cloud *:9095 running (3m) 43s ago 2h 56.7M - 2.33.4 514e6a882f6e 7873cd72c9f0
bash
[root@ceph1 ~ 15:06:38]# ceph orch ls mon
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
mon 3/3 5m ago 55m label:_admin
[root@ceph1 ~ 15:07:13]# ceph orch apply mon --unmanaged=true
Scheduled mon update...
[root@ceph1 ~ 15:07:24]# ceph orch ls mon
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
mon 3/5 6m ago 8s <unmanaged>
[root@ceph1 ~ 15:07:32]# ceph orch apply mon --unmanaged=false
Scheduled mon update...
[root@ceph1 ~ 15:07:49]# ceph orch ls mon
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
mon 3/5 6m ago 6s count:5
[root@ceph1 ~ 15:07:55]# ceph orch apply mon --placement="label:_admin"
Scheduled mon update...
[root@ceph1 ~ 15:08:06]# ceph orch ls mon
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
mon 3/3 6m ago 6s label:_admin
[root@ceph1 ~ 15:08:13]# ceph orch apply crash --unmanaged=true
Scheduled crash update...
[root@ceph1 ~ 15:08:24]# ceph orch ls crash
NAME PORTS RUNNING REFRESHED AGE PLACEMENT
crash 3/3 7m ago 7s <unmanaged>
[root@ceph1 ~ 15:08:32]# ceph orch ps | grep crash
crash.ceph1 ceph1.my.cloud running (48m) 7m ago 3h 6643k - 16.2.15 3c4eff6082ae 6dfe04fecbd9
crash.ceph2 ceph2.my.cloud running (58m) 7m ago 58m 6643k - 16.2.15 3c4eff6082ae c3b5040a5452
crash.ceph3 ceph3.my.cloud running (58m) 7m ago 58m 6639k - 16.2.15 3c4eff6082ae a613c9b30e6b
[root@ceph1 ~ 15:08:40]# ceph orch daemon rm crash.ceph1
Removed crash.ceph1 from host 'ceph1.my.cloud'
[root@ceph1 ~ 15:09:01]# ceph orch ps | grep crash
crash.ceph2 ceph2.my.cloud running (59m) 7m ago 59m 6643k - 16.2.15 3c4eff6082ae c3b5040a5452
crash.ceph3 ceph3.my.cloud running (59m) 7m ago 59m 6639k - 16.2.15 3c4eff6082ae a613c9b30e6b
[root@ceph1 ~ 15:09:13]# ceph orch rm crash
Removed service crash
[root@ceph1 ~ 15:09:21]# ceph orch ls crash
No services reported
[root@ceph1 ~ 15:09:31]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.17537 root default
-7 0.05846 host ceph1
2 ssd 0.01949 osd.2 up 1.00000 1.00000
5 ssd 0.01949 osd.5 up 1.00000 1.00000
8 ssd 0.01949 osd.8 up 1.00000 1.00000
-5 0.05846 host ceph2
0 ssd 0.01949 osd.0 up 1.00000 1.00000
3 ssd 0.01949 osd.3 up 1.00000 1.00000
6 ssd 0.01949 osd.6 up 1.00000 1.00000
-3 0.05846 host ceph3
1 ssd 0.01949 osd.1 up 1.00000 1.00000
4 ssd 0.01949 osd.4 up 1.00000 1.00000
7 ssd 0.01949 osd.7 up 1.00000 1.00000
[root@ceph1 ~ 15:09:52]# ceph -s | grep id
id: 96b28050-5980-11f1-8411-000c2993506d
[root@ceph1 ~ 15:10:03]# ls -l /var/lib/ceph/96b28050-5980-11f1-8411-000c2993506d/osd.0/block
[root@ceph1 ~ 15:13:50]# lsblk | grep -B1 59879d2ad99c
[root@ceph1 ~ 15:18:47]# lsblk | grep -B1 96b28050-5980-11f1-8411-000c2993506d
# 禁用 osd 服务自动扩展
[root@ceph1 ~]# ceph orch apply osd --all-available-devices --unmanaged=true
[root@ceph1 ~]# ceph orch ls osd
NAME
PORTS RUNNING REFRESHED AGE PLACEMENT
osd.all-available-devices
# 第一种（先标记out，再删除）
[root@ceph1 ~]# ceph osd out osd.0
[root@ceph1 ~]# ceph orch osd rm 0
9 7m ago
6s
<unmanaged>
# 删除device上数据
# 删除前lsblk看下sdb
[root@ceph1 ~]# ceph orch device zap ceph1.my.cloud /dev/sdb --force
osd0管理的是/dev/sdb还是啥
zap successful for /dev/sdb on ceph1.my.cloud
# 确认结果
[root@ceph1 ~]# ceph orch device ls
[root@ceph1 ~]# ceph orch device ls | grep ceph1.*sdb
ceph1.laogao.cloud /dev/sdb hdd
Yes
72s ago
8:16
[root@ceph1 ~]# lsblk
[root@ceph1 ~]# lsblk /dev/sdb
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sdb
0 20G 0 disk
# 第二种 直接删除加格式化磁盘
[root@ceph1 ~]# ceph osd out osd.3
[root@ceph1 ~]# ceph orch osd rm 3 --force --zap
20.0G
# 添加回来
[root@ceph1 ~]# ceph orch apply osd --all-available-devices
#RUNNING重新变为9
[root@ceph1 ~]# ceph orch ls osd
NAME
PORTS RUNNING REFRESHED AGE PLACEMENTosd删除
bash
[root@ceph1 ~]# ceph osd tree
ID CLASS WEIGHT TYPE NAME -1 -3
0.17537 root default
0.05846
ceph1下的硬盘
0
hdd 0.01949
#osd.0对应哪块磁盘
3
hdd 0.01949
6 -5
2
4
7 -7
1
5
8
hdd 0.01949
0.05846
hdd 0.01949
hdd 0.01949
hdd 0.01949
0.05846
hdd 0.01949
hdd 0.01949
STATUS REWEIGHT PRI-AFF
host ceph1
osd.0
osd.3
osd.6
host ceph2
osd.2
osd.4
up
up
up
1.00000 1.00000
1.00000 1.00000
1.00000 1.00000
up
up
osd.7
host ceph3
osd.1
osd.5
hdd 0.01949
osd.8
up
up
up
up
1.00000 1.00000
1.00000 1.00000
1.00000 1.00000
1.00000 1.00000
1.00000 1.00000
1.00000 1.00000
#看不出来
# 获取集群id
[root@ceph1 ~]# ceph -s | grep id
id: 2faf683a-7cbf-11f0-b5ba-000c29e0ad0e
# 登录到ceph1上确认osd.0使用的块设备
[root@ceph1 ~]# ls -l /var/lib/ceph/2faf683a-7cbf-11f0-b5ba
000c29e0ad0e/osd.0/block
lrwxrwxrwx 1 ceph ceph 93 Aug 19 14:01 /var/lib/ceph/2faf683a-7cbf-11f0-b5ba
000c29e0ad0e/osd.0/block -> /dev/ceph-c92942fb-f959-4255-b8e6-751fab70fa79/osd
block-2ed79b2f-d825-4829-b4b0-59879d2ad99c
# 59879d2ad99c是块设备名称最后一串字符
[root@ceph1 ~]# lsblk | grep -B1 59879d2ad99c
sdb
8:16 0 20G 0 disk
└─ceph--c92942fb--f959--4255--b8e6--751fab70fa79-osd--block--2ed79b2f--d825--4829--b4b0--59879d2ad99c 253:4 0 20G 0 lvm
# 确认osd.0对应sdb
# 用下面命令也可以
[root@ceph1 ~]# ceph osd metadata 0 #0是osd编号
[root@ceph1 ~]# ceph osd metadata 0
{
"id": 0,
"arch": "x86_64",
"back_addr": "
[v2:192.168.108.11:6802/914648970,v1:192.168.108.11:6803/914648970]",
"back_iface": "",
"bluefs": "1",
"bluefs_dedicated_db": "0",
"bluefs_dedicated_wal": "0",
"bluefs_single_shared_device": "1",
"bluestore_bdev_access_mode": "blk",
"bluestore_bdev_block_size": "4096",
"bluestore_bdev_dev_node": "/dev/dm-2",
"bluestore_bdev_devices": "sdb",
"bluestore_bdev_driver": "KernelDevice",
"bluestore_bdev_partition_path": "/dev/dm-2",
"bluestore_bdev_rotational": "1",
"bluestore_bdev_size": "21470642176",
"bluestore_bdev_support_discard": "0",
"bluestore_bdev_type": "hdd",
"bluestore_min_alloc_size": "4096",
"ceph_release": "pacific",
"ceph_version": "ceph version 16.2.15
(618f440892089921c3e944a991122ddc44e60516) pacific (stable)",
"ceph_version_short": "16.2.15",
"ceph_version_when_created": "ceph version 16.2.15
(618f440892089921c3e944a991122ddc44e60516) pacific (stable)",
"container_hostname": "ceph1.laogao.cloud",
"container_image":
"quay.io/ceph/ceph@sha256:6ba107eb55617994a9e6ed49fb938828c2ed3121aa19ceeffbf8e28
608535d94",
"cpu": "Intel(R) Core(TM) Ultra 9 185H",
"created_at": "2026-03-27T06:01:45.491319Z",
"default_device_class": "hdd",
"device_ids": "",
"device_paths": "sdb=/dev/disk/by-path/pci-0000:00:10.0-scsi-0:0:1:0",
"devices": "sdb",
"distro": "centos",
"distro_description": "CentOS Stream 8",
"distro_version": "8",
"front_addr": "
[v2:192.168.108.11:6800/914648970,v1:192.168.108.11:6801/914648970]",
"front_iface": "",
"hb_back_addr": "
[v2:192.168.108.11:6806/914648970,v1:192.168.108.11:6807/914648970]",
"hb_front_addr": "
[v2:192.168.108.11:6804/914648970,v1:192.168.108.11:6805/914648970]",
"hostname": "ceph1.my.cloud",
"journal_rotational": "1",
"kernel_description": "#1 SMP Thu May 30 04:13:58 UTC 2024",
"kernel_version": "4.18.0-553.6.1.el8.x86_64",
"mem_swap_kb": "8245244",
"mem_total_kb": "7849664",
"network_numa_unknown_ifaces": "back_iface,front_iface",
"objectstore_numa_unknown_devices": "sdb",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-0",
"osd_objectstore": "bluestore",
"osdspec_affinity": "all-available-devices",
"rotational": "1"
}主机删除
bash
[root@ceph1 ~]# for service in $(ceph orch ls |grep -v -e NAME -e osd| awk
'{print $1}');do ceph orch apply $service --unmanaged=true;done
[root@ceph1 ~]# ceph orch apply osd --all-available-devices --unmanaged=true
其次，删除主机上运行的服务。
# 查看ceph2上运行的daemon
[root@ceph1 ~]# ceph orch ps |grep ceph2 |awk '{print $1}'
crash.ceph2
mgr.ceph2.oetbal
mon.ceph2
node-exporter.ceph2
osd.2
osd.4
osd.7
# 删除相应 daemon
[root@ceph1 ~]# for daemon in $(ceph orch ps |grep ceph2 |awk '{print $1}');do
ceph orch daemon rm $daemon --force;done
# 手动清理crush信息
[root@ceph1 ~]# ceph osd crush rm osd.2
[root@ceph1 ~]# ceph osd crush rm osd.4
[root@ceph1 ~]# ceph osd crush rm osd.7
[root@ceph1 ~]# ceph osd crush rm ceph2
[root@ceph1 ~]# ceph osd rm 2 4 7
# 清理磁盘数据
[root@ceph1 ~]# ceph orch device zap ceph2.laogao.cloud /dev/sdb --force
[root@ceph1 ~]# ceph orch device zap ceph2.laogao.cloud /dev/sdc --force
[root@ceph1 ~]# ceph orch device zap ceph2.laogao.cloud /dev/sdd --force
然后，删除主机。
[root@ceph1 ~]# ceph orch host rm ceph2
[root@ceph1 ~]# ceph orch host ls
HOST
ADDR
#查看现象ceph2被移除
LABELS STATUS
ceph1.laogao.cloud 192.168.108.11 _admin
ceph3.laogao.cloud 192.168.108.13 _admin
2 hosts in cluster
#最后，删除ceph2中相应ceph遗留文件。
[root@ceph2 ~]# rm -rf /var/lib/ceph
[root@ceph2 ~]# rm -rf /etc/ceph /etc/systemd/system/ceph*
[root@ceph2 ~]# rm -rf /var/log/cephceph orch daemon rm $daemon --force;done
手动清理crush信息
root@ceph1 \~# ceph osd crush rm osd.2
root@ceph1 \~# ceph osd crush rm osd.4
root@ceph1 \~# ceph osd crush rm osd.7
root@ceph1 \~# ceph osd crush rm ceph2
root@ceph1 \~# ceph osd rm 2 4 7
清理磁盘数据
root@ceph1 \~# ceph orch device zap ceph2.laogao.cloud /dev/sdb --force
root@ceph1 \~# ceph orch device zap ceph2.laogao.cloud /dev/sdc --force
root@ceph1 \~# ceph orch device zap ceph2.laogao.cloud /dev/sdd --force
然后，删除主机。
root@ceph1 \~# ceph orch host rm ceph2
root@ceph1 \~# ceph orch host ls
HOST
ADDR
#查看现象ceph2被移除
LABELS STATUS
ceph1.laogao.cloud 192.168.108.11 _admin
ceph3.laogao.cloud 192.168.108.13 _admin
2 hosts in cluster
#最后，删除ceph2中相应ceph遗留文件。
root@ceph2 \~# rm -rf /var/lib/ceph
root@ceph2 \~# rm -rf /etc/ceph /etc/systemd/system/ceph*
root@ceph2 \~# rm -rf /var/log/ceph