docker部署ceph集群

安装之前的准备

bash 复制代码

我的操作系统
[root@ceph02 ~]# uname -a
Linux ceph02 6.6.0-105.0.0.111.oe2403sp2.aarch64 #1 SMP Wed Aug 13 16:24:53 CST 2025 aarch64 aarch64 aarch64 GNU/Linux
[root@ceph02 ~]# cat /etc/os-release
NAME="openEuler"
VERSION="24.03 (LTS-SP2)"
ID="openEuler"
VERSION_ID="24.03"
PRETTY_NAME="openEuler 24.03 (LTS-SP2)"
ANSI_COLOR="0;31"

通过网盘分享的文件：arm64 ceph安装包

链接: https://pan.baidu.com/s/1p4xr1uNzP8sRxdxiFiU-zA?pwd=bd6k 提取码: bd6k

导入镜像安装cephadm

bash 复制代码

docker load -i ceph-v19.2.6.tar
docker load -i grafana.tar
docker load -i prometheus.tar
docker load -i node-exporter.tar
docker load -i alertmanager.tar
mv cephadm /usr/sbin/ && chmod +x /usr/sbin/cephadm

一、先过掉 cephadm 的时间检查

openEuler 24.03 上 cephadm 认不出 chronyd，即使 chronyd 在运行也会报 No time sync。必须先创建一个 cephadm 能识别的兼容服务。

在 ceph01 上执行：

bash 复制代码

cat > /etc/systemd/system/chrony.service <<'EOF'
[Unit]
Description=Chrony NTP compat shim for cephadm
After=chronyd.service
Requires=chronyd.service

[Service]
Type=oneshot
ExecStart=/bin/true
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable --now chrony.service

二、安装ceph集群

bash 复制代码

cephadm --image quay.io/ceph/ceph:v19.2.6   bootstrap --mon-ip 10.90.11.16   --skip-prepare-host  --skip-pull

三、加节点

bash 复制代码

ssh-copy-id -f -i /etc/ceph/ceph.pub root@ceph02
ssh-copy-id -f -i /etc/ceph/ceph.pub root@ceph03
#进入容器
export CEPHADM_IMAGE=quay.io/ceph/ceph:v19.2.6
cephadm shell
ceph config set mgr mgr/cephadm/use_repo_digest false #关闭 repo digest（离线部署关键,使用本地镜像）
ceph orch host add ceph02 10.90.11.17
ceph orch host add ceph03 10.90.11.18
ceph orch apply mon --placement="ceph01,ceph02,ceph03"
ceph orch apply mgr --placement="ceph01,ceph02,ceph03"
ceph -s
ceph orch host ls
#删除不必要的组件
ceph orch rm alertmanager
ceph orch rm grafana
ceph orch rm prometheus
ceph orch rm node-exporter

确认离线镜像配置

bash 复制代码

cephadm --image quay.io/ceph/ceph:v19.2.6 shell -- \
  ceph config set global container_image quay.io/ceph/ceph:v19.2.6

四、增加osd

bash 复制代码

 wipefs -a /dev/nvme0n1 #擦除 /dev/nvme0n1这块磁盘上所有已知的文件系统
 #进入容器
[root@ceph01 ~]# export CEPHADM_IMAGE=quay.io/ceph/ceph:v19.2.6
[root@ceph01 ~]# cephadm shell
Inferring fsid 3924d610-c7b0-11f1-b569-fc119386944e
Inferring config /var/lib/ceph/3924d610-c7b0-11f1-b569-fc119386944e/mon.ceph01/config
[ceph: root@ceph01 /]# ceph -s
  cluster:
    id:     3924d610-c7b0-11f1-b569-fc119386944e
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum ceph01,ceph03,ceph02 (age 17h)
    mgr: ceph01.jsgdms(active, since 18h), standbys: ceph03.lfeosf, ceph02.owsuba
    osd: 3 osds: 3 up (since 17h), 3 in (since 17h)

  data:
    pools:   1 pools, 1 pgs
    objects: 2 objects, 577 KiB
    usage:   91 MiB used, 5.2 TiB / 5.2 TiB avail
    pgs:     1 active+clean