Ceph 集群手动部署

Ceph 集群手动部署

本文在 Rocky8.8 系统搭建三节点 Ceph Pacific 集群，依次部署 MON、MGR、OSD、MDS、RGW 核心组件。提前优化系统环境，配置密钥安全认证，开启监控可视化面板，搭建文件与对象存储服务，完成挂载及文件上传测试，成功构建可用的分布式存储集群。

节点规划

主机名	IP地址	角色
ceph01.example.com	172.18.0.10/24	mon、mgr、osd、mds、rgw
ceph02.example.com	172.18.0.20/24	mon、mgr、osd、mds、rgw
ceph03.example.com	172.18.0.30/24	mon、mgr、osd、mds、rgw

操作系统：Rocky Linux release 8.8 (Green Obsidian)

一、前期准备

所有节点统一执行

1. 替换阿里云YUM源

sed -e 's|^mirrorlist=|#mirrorlist=|g' \
-e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
-i.bak /etc/yum.repos.d/Rocky-.repo

dnf makecache

2. 关闭防火墙与SELinux

systemctl disable --now firewalld.service
setenforce 0
# 永久关闭
vim /etc/sysconfig/selinux
# 修改 SELINUX=disabled

3. 配置时间同步

echo "server ntp.aliyun.com iburst" >> /etc/chronyd.conf
systemctl enable --now chronyd
chronyc sources

4. 主机名IP映射

vim /etc/hosts
# 写入内容
172.18.0.10 ceph01.example.com ceph01
172.18.0.20 ceph02.example.com ceph02
172.18.0.30 ceph03.example.com ceph03

5. 安装Ceph Pacific版本

yum install centos-release-ceph-pacific -y
yum install ceph -y

二、部署MON监控节点

1. 生成集群UUID

uuidgen
# 示例输出
eb6d6dab-6854-4663-ba7f-eadf2758e681

2. 编写集群配置文件

vim /etc/ceph/ceph.conf

[global]
fsid=eb6d6dab-6854-4663-ba7f-eadf2758e681
mon_initial_members=ceph01,ceph02,ceph03
mon_host=172.18.0.10,172.18.0.20,172.18.0.30
public_network=172.18.0.0/24

auth_cluster_required=cephx
auth_service_required=cephx
auth_client_required=cephx

osd_pool_default_size=3
osd_pool_default_min_size=2
osd_pool_default_pg_num=128
osd_pool_default_pgp_num=128

storage_type=bluestore
osd_objectstore=bluestore
osd_mkfs_type=xfs
bluestore_block_size=5628755968

3. 生成各类密钥环

# 生成monitor密钥
ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'

# 生成管理员密钥
ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'

# 生成OSD引导密钥
ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'

# 密钥导入合并
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring

# 授权属主
chown ceph:ceph /tmp/ceph.mon.keyring

4. 生成monmap集群视图

monmaptool --create --add ceph01 172.18.0.10 --add ceph02 172.18.0.20 --add ceph03 172.18.0.30 --fsid eb6d6dab-6854-4663-ba7f-eadf2758e681 /tmp/monmap

5. 分发配置与密钥文件

scp /etc/ceph/ceph.conf root@ceph02:/etc/ceph/
scp /etc/ceph/ceph.conf root@ceph03:/etc/ceph/

scp /tmp/monmap root@ceph02:/tmp/
scp /tmp/monmap root@ceph03:/tmp/

scp /tmp/ceph.mon.keyring root@ceph02:/tmp/
scp /tmp/ceph.mon.keyring root@ceph03:/tmp/

scp /etc/ceph/ceph.client.admin.keyring root@ceph02:/etc/ceph/
scp /etc/ceph/ceph.client.admin.keyring root@ceph03:/etc/ceph/

6. 所有节点初始化MON数据

# ceph01执行
sudo -u ceph mkdir /var/lib/ceph/mon/ceph-ceph01
ceph-mon --mkfs -i ceph01 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

# ceph02执行
sudo -u ceph mkdir /var/lib/ceph/mon/ceph-ceph02
ceph-mon --mkfs -i ceph02 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

# ceph03执行
sudo -u ceph mkdir /var/lib/ceph/mon/ceph-ceph03
ceph-mon --mkfs -i ceph03 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

# 统一授权目录
chown ceph.ceph -R /var/lib/ceph

7. 启动MON服务并排查告警

# 各节点启动服务
systemctl enable --now ceph-mon@ceph01
systemctl enable --now ceph-mon@ceph02
systemctl enable --now ceph-mon@ceph03

# 查看集群状态
ceph -s

# 问题1：未启用msgr2协议
ceph mon enable-msgr2

# 问题2：关闭不安全全局ID回收
ceph config set mon auth_allow_insecure_global_id_reclaim false

三、部署MGR管理节点

所有节点执行

1. 创建密钥目录

sudo -u ceph mkdir /var/lib/ceph/mgr/ceph-$(hostname -s)

2. 生成MGR密钥

ceph auth get-or-create mgr.$(hostname -s) mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-$(hostname -s)/keyring

3. 启动MGR服务

systemctl enable --now ceph-mgr@ceph01
systemctl enable --now ceph-mgr@ceph02
systemctl enable --now ceph-mgr@ceph03

4. 部署Dashboard可视化面板

# 安装面板组件
yum install ceph-mgr-dashboard -y

# 生成自签名证书
openssl req -new -nodes -x509 -subj "/O=IT/CN=ceph-mgr-dashboard" -days 3650 -keyout dashboard.key -out dashboard.crt -extensions v3_ca

# 导入证书
ceph dashboard set-ssl-certificate -i dashboard.crt
ceph dashboard set-ssl-certificate-key -i dashboard.key

# 配置访问地址端口
ceph config set mgr mgr/dashboard/server_addr 0.0.0.0
ceph config set mgr mgr/dashboard/ssl_server_port 8443

# 关闭RGW证书校验
ceph dashboard set-rgw-api-ssl-verify false

# 创建登录用户
echo redhat123 > pass.txt
ceph dashboard ac-user-create admin administrator -i pass.txt

# 关闭SSL改用HTTP访问
ceph config set mgr mgr/dashboard/ssl false

# 启停面板模块
ceph config module disable dashboard
ceph config module enable dashboard

四、部署OSD存储节点

1. 分发OSD引导密钥

# ceph01执行分发
scp /var/lib/ceph/bootstrap-osd/ceph.keyring root@ceph02:/var/lib/ceph/bootstrap-osd/
scp /var/lib/ceph/bootstrap-osd/ceph.keyring root@ceph03:/var/lib/ceph/bootstrap-osd/

# 所有节点授权文件
chown ceph.ceph /var/lib/ceph/bootstrap-osd/ceph.keyring

2. 创建Bluestore类型OSD

# 单条命令创建
ceph-volume lvm create --data /dev/sdb --bluestore

# 拆分两步创建
ceph-volume lvm prepare --data /dev/sdb
ceph-volume lvm activate {ID} {FSID}

# 查看OSD信息
ceph-volume lvm list

3. 查看集群状态

ceph -s
ceph osd ls

五、部署MDS文件系统服务

所有节点执行

1. 新建密钥目录

sudo -u ceph mkdir /var/lib/ceph/mds/ceph-$(hostname -s)

2. 生成MDS密钥

ceph auth get-or-create mds.$(hostname -s) osd "allow rwx" mds "allow" mon "allow profile mds" > /var/lib/ceph/mds/ceph-$(hostname -s)/keyring
chown ceph.ceph -R /var/lib/ceph/mds/

3. 启动MDS服务

systemctl start ceph-mds@ceph01
systemctl start ceph-mds@ceph02
systemctl start ceph-mds@ceph03

4. 创建文件系统测试

# 创建元数据、数据存储池
ceph osd pool create fspool_meta
ceph osd pool create fspool_data

# 初始化文件系统
ceph fs new fs01 fspool_meta fspool_data

# 查看文件系统状态
ceph fs status

# 创建授权访问用户
ceph fs authorize fs01 client.wangwu / rw

# 客户端挂载测试
mount -t ceph ceph01:/ /media/ -o name=wangwu,fs=fs01,secret=AQB3ryJl1Z2QMRAAH1C/qRaBGIKKBWET5lM1eg==

六、部署RGW对象存储网关

1. 安装RGW组件

所有节点执行

yum install ceph-radosgw -y

2. 生成网关密钥

# ceph01执行
ceph auth get-or-create client.rgw.ceph01 osd 'allow rwx' mon 'allow rwx' >> /etc/ceph/ceph.client.radosgw.keyring
ceph auth get-or-create client.rgw.ceph02 osd 'allow rwx' mon 'allow rwx' >> /etc/ceph/ceph.client.radosgw.keyring
ceph auth get-or-create client.rgw.ceph03 osd 'allow rwx' mon 'allow rwx' >> /etc/ceph/ceph.client.radosgw.keyring

chown ceph:ceph /etc/ceph/ceph.client.radosgw.keyring

# 分发密钥
scp /etc/ceph/ceph.client.radosgw.keyring root@ceph02:/etc/ceph/
scp /etc/ceph/ceph.client.radosgw.keyring root@ceph03:/etc/ceph/

# 节点授权
chown ceph:ceph /etc/ceph/ceph.client.radosgw.keyring

3. 追加RGW配置

vim /etc/ceph/ceph.conf

[client.rgw.ceph01]
host=ceph01
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw frontends = civetweb port=172.18.0.10:8080

[client.rgw.ceph02]
host=ceph02
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw frontends = civetweb port=172.18.0.20:8080

[client.rgw.ceph03]
host=ceph03
keyring=/etc/ceph/ceph.client.radosgw.keyring
log file=/var/log/radosgw/client.radosgw.gateway.log
rgw frontends = civetweb port=172.18.0.30:8080

# 分发配置文件
scp /etc/ceph/ceph.conf root@ceph02:/etc/ceph/
scp /etc/ceph/ceph.conf root@ceph03:/etc/ceph/

4. 创建日志目录

sudo -u ceph mkdir /var/log/radosgw/

5. 启动RGW网关服务

systemctl enable --now ceph-radosgw@rgw.ceph01
systemctl enable --now ceph-radosgw@rgw.ceph02
systemctl enable --now ceph-radosgw@rgw.ceph03

6. 创建S3访问用户

radosgw-admin user create --uid=user1 --access-key=123 --secret=456 --email user1@example.com --display-name user1

7. S3对象存储功能测试

# 安装客户端工具
yum install epel-release -y
yum install aws -y

# 配置访问密钥
aws configure --profile=ceph

# 创建存储桶
aws --profile=ceph --endpoint-url=http://172.18.0.10:8080 s3 mb s3://bucket1

# 上传测试文件
aws --profile=ceph --endpoint-url http://172.18.0.10:8080 s3 cp /etc/passwd s3://bucket1/passwd