ceph镜像

2301_773643622026-06-10 17:08

ceph镜像

rbd缓存参数

bash 复制代码 
[root@ceph1 ~ 09:59:20]# ceph config ls | grep rbd_cache
rbd_cache
rbd_cache_policy
rbd_cache_writethrough_until_flush
rbd_cache_size
rbd_cache_max_dirty
rbd_cache_target_dirty
rbd_cache_max_dirty_age
rbd_cache_max_dirty_object
rbd_cache_block_writes_upfront
[root@ceph1 ~ 10:32:38]# for arg in $(ceph config ls|grep rbd_cache)
> do
> echo -n "$arg: "
> ceph config get client $arg
> done
rbd_cache: true
rbd_cache_policy: writearound
rbd_cache_writethrough_until_flush: true
rbd_cache_size: 33554432
rbd_cache_max_dirty: 25165824
rbd_cache_target_dirty: 16777216
rbd_cache_max_dirty_age: 1.000000
rbd_cache_max_dirty_object: 0
rbd_cache_block_writes_upfront: false
[root@ceph1 ~ 10:33:07]# ceph config set client rbd_cache_policy writethrough
[root@ceph1 ~ 10:34:12]# ceph config get client rbd_cache_policy
writethrough
[root@ceph1 ~ 10:34:22]# ceph config set global rbd_cache_policy writethrough
[root@ceph1 ~ 10:34:31]# ceph config get client rbd_cache_policy
writethrough

rbd镜像布局

bash 复制代码 
[root@ceph1 ~ 10:34:36]# rbd info images_pool/webapp1 --id rbd
rbd image 'webapp1':
        size 1 GiB in 256 objects
        order 22 (4 MiB objects)
        snapshot_count: 0
        id: 14e4894cdd9f3
        block_name_prefix: rbd_data.14e4894cdd9f3
        format: 2
        features: layering
        op_features:
        flags:
        create_timestamp: Fri May 29 17:24:30 2026
        access_timestamp: Fri May 29 17:24:30 2026
        modify_timestamp: Fri May 29 17:24:30 2026

rbd镜像阶数

bash 复制代码 
[root@ceph1 ~ 10:35:36]# rbd help create
usage: rbd create [--pool <pool>] [--namespace <namespace>] [--image <image>]
                  [--image-format <image-format>] [--new-format]
                  [--order <order>] [--object-size <object-size>]
                  [--image-feature <image-feature>] [--image-shared]
                  [--stripe-unit <stripe-unit>]
                  [--stripe-count <stripe-count>] [--data-pool <data-pool>]
                  [--mirror-image-mode <mirror-image-mode>]
                  [--journal-splay-width <journal-splay-width>]
                  [--journal-object-size <journal-object-size>]
                  [--journal-pool <journal-pool>]
                  [--thick-provision] --size <size> [--no-progress]
                  <image-spec>

Create an empty image.

Positional arguments
  <image-spec>              image specification
                            (example: [<pool-name>/[<namespace>/]]<image-name>)

Optional arguments
  -p [ --pool ] arg         pool name
  --namespace arg           namespace name
  --image arg               image name
  --image-format arg        image format [default: 2]
  --object-size arg         object size in B/K/M [4K <= object size <= 32M]
  --image-feature arg       image features
                            [layering(+), exclusive-lock(+*), object-map(+*),
                            deep-flatten(+-), journaling(*)]
  --image-shared            shared image
  --stripe-unit arg         stripe unit in B/K/M
  --stripe-count arg        stripe count
  --data-pool arg           data pool
  --mirror-image-mode arg   mirror image mode [journal or snapshot]
  --journal-splay-width arg number of active journal objects
  --journal-object-size arg size of journal objects [4K <= size <= 64M]
  --journal-pool arg        pool for journal objects
  --thick-provision         fully allocate storage and zero image
  -s [ --size ] arg         image size (in M/G/T) [default: M]
  --no-progress             disable progress output

Image Features:
  (*) supports enabling/disabling on existing images
  (-) supports disabling-only on existing images
  (+) enabled by default for new images if features not specified

rbd镜像格式

bash 复制代码 
[root@ceph1 ~ 10:38:05]# rbd create --stripe-unit=1M --stripe-count=8 --size 1G images_pool/webapp2 --id rbd
[root@ceph1 ~ 10:38:25]# rbd info images_pool/webapp2 --id rbd
rbd image 'webapp2':
        size 1 GiB in 256 objects
        order 22 (4 MiB objects)
        snapshot_count: 0
        id: 170349b545a2f
        block_name_prefix: rbd_data.170349b545a2f
        format: 2
        features: layering, striping, exclusive-lock, object-map, fast-diff, deep-flatten
        op_features:
        flags:
        create_timestamp: Mon Jun  1 10:38:25 2026
        access_timestamp: Mon Jun  1 10:38:25 2026
        modify_timestamp: Mon Jun  1 10:38:25 2026
        stripe unit: 1 MiB
        stripe count: 8

rbd命令管理镜像

bash 复制代码 
#status:查看哪些客户端在使用镜像
[root@ceph1 ~ 10:38:32]# rbd status images_pool/webapp1 --id rbd
Watchers:
        watcher=192.168.108.10:0/537007787 client.94153 cookie=18446462598732840961
[root@ceph1 ~ 10:40:47]# rbd status images_pool/webapp2 --id rbd
Watchers: none
#du：扩展缩减（缩减尽量别用）镜像
[root@ceph1 ~ 10:40:53]# rbd resize images_pool/webapp2 --size 2G --id rbd
Resizing image: 100% complete...done.
[root@ceph1 ~ 10:42:09]# rbd du images_pool/webapp2
NAME     PROVISIONED  USED
webapp2        2 GiB   0 B
[root@client ~]# rbd resize images_pool/webapp1 --size 2G --id rbd
Resizing image: 100% complete...done.
[root@client ~]# lsblk /dev/rbd0
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
rbd0 252:0 0 2G 0 disk /webapp/webapp1
[root@client ~]# df -h /dev/rbd0
Filesystem Size Used Avail Use% Mounted on
/dev/rbd0 1014M 40M 975M 4% /webapp/webapp1
[root@client ~]# xfs_growfs /webapp/webapp1/
[root@client ~]# df -h /webapp/webapp1/
Filesystem Size Used Avail Use% Mounted on
/dev/rbd0 2.0G 48M 2.0G 3% /webapp/webapp1
[root@client ~]# rbd resize images_pool/webapp2 --size 1G --id rbd
rbd: shrinking an image is only allowed with the --allow-shrink flag
[root@client ~]# rbd resize images_pool/webapp2 --size 1G --allow-shrink --id rbd
Resizing image: 100% complete...done.
[root@client ~]# rbd du images_pool/webapp2 --id rbd
NAME PROVISIONED USED
webapp2 1 GiB 0 B
#rename和mv用于重命名镜像
#不支持跨池命名
[root@ceph1 ~ 10:42:58]# ceph osd pool create images_pool_2 32 32
pool 'images_pool_2' created
[root@ceph1 ~ 10:44:21]# rbd pool init images_pool_2
[root@ceph1 ~ 10:44:29]# rbd rename images_pool/webapp2 images_pool/webapp02
[root@ceph1 ~ 10:44:37]# rbd ls images_pool
webapp02
webapp1
[root@ceph1 ~ 10:44:43]# rbd mv images_pool/webapp02 images_pool/webapp2
[root@ceph1 ~ 10:44:52]# rbd ls images_pool
webapp1
webapp2
[root@ceph1 ~ 10:44:56]# rbd rename images_pool/webapp2 images_pool_2/webapp02
rbd: mv/rename across pools not supported
source pool: images_pool dest pool: images_pool_2
#cp用于复制镜像
[root@ceph1 ~ 10:45:04]# ceph osd lspools
5 images_pool
6 device_health_metrics
7 images_pool_2
[root@ceph1 ~ 10:46:31]# rbd  ls images_pool
webapp1
webapp2
[root@ceph1 ~ 10:46:45]# rbd ls images_pool_2
[root@ceph1 ~ 10:47:13]# rbd cp images_pool/webapp2 images_pool_2/webapp2
Image copy: 100% complete...done.
[root@ceph1 ~ 10:47:30]# rbd  ls images_pool
webapp1
webapp2
[root@ceph1 ~ 10:47:35]# rbd ls images_pool_2
webapp2
#trash用于垃圾箱管理镜像
[root@ceph1 ~ 10:51:44]# rbd trash mv images_pool/webapp2
[root@ceph1 ~ 11:13:21]# rbd  ls images_pool
webapp1
[root@ceph1 ~ 11:13:28]# rbd trash ls images_pool
170349b545a2f webapp2
[root@ceph1 ~ 11:13:36]# rbd trash restore -p images_pool 170349b545a2f
[root@ceph1 ~ 11:13:56]# rbd trash ls images_pool
[root@ceph1 ~ 11:14:00]# rbd  ls images_pool
webapp1
webapp2
[root@ceph1 ~ 11:14:23]# rbd trash mv images_pool/webapp2
[root@ceph1 ~ 11:14:34]# rbd trash rm -p images_pool 170349b545a2f
Removing image: 100% complete...done.
[root@ceph1 ~ 11:14:56]# rbd trash ls images_pool
[root@ceph1 ~ 11:15:03]# rbd  ls images_pool
webapp1
#rm用于删除镜像，不经过回收站
[root@ceph1 ~ 11:15:08]# rbd rm images_pool_2/webapp2
Removing image: 100% complete...done.
[root@ceph1 ~ 11:15:59]# rbd  ls images_pool_2

rados块设备快照

bash 复制代码 
#集群中rbd镜像默认启用功能
[root@ceph1 ~ 11:16:09]# ceph config get client rbd_default_features
layering,exclusive-lock,object-map,fast-diff,deep-flatten
#永久功能无法禁用启用
[root@ceph1 ~ 11:17:00]# rbd feature disable images_pool/webapp1 object-map
rbd: failed to update image features: (22) Invalid argument
2026-06-01T11:17:53.663+0800 7fec62c783c0 -1 librbd::Operations: one or more requested features are already disabled

[root@ceph1 ~ 11:18:01]# rbd feature enable images_pool/webapp1 object-map
rbd: failed to update image features: (22) Invalid argument
2026-06-01T11:18:26.336+0800 7fc6e64d9700 -1 librbd::EnableFeaturesRequest: cannot enable object-map. exclusive-lock must be enabled before enabling object-map.

rbd快照

注意： 在拍摄快照前，一定要完成以下任一操作：

  1. 卸载文件系统。
  2. 使用 fsfreeze命令冻结文件系统，只允许读操作。

使用 rbd snap create 命令，创建 Ceph 块设备的快照。

bash 复制代码 
[root@client ~ 11:22:12]# export CEPH_ARGS='--id=rbd'
[root@client ~ 11:22:20]# rbd create images_pool/webapp --size 2G
rbd: create error: (17) File exists
2026-06-01T11:22:25.075+0800 7f8cb2fff3c0 -1 librbd: rbd image webapp already exists

[root@client ~ 11:22:25]# rbd map images_pool/webapp
/dev/rbd1
[root@client ~ 11:22:38]# mkfs.xfs /dev/rbd/images_pool/webapp
mkfs.xfs: /dev/rbd/images_pool/webapp appears to contain an existing filesystem (xfs).
mkfs.xfs: Use the -f option to force overwrite.
[root@client ~ 11:22:48]# mkdir /webapp/webapp
[root@client ~ 11:22:57]# # mount /dev/rbd/images_pool/webapp /webapp/webapp
[root@client ~ 11:23:04]# echo Hello World > /webapp/webapp/index.html
[root@client ~ 11:23:10]# cat /webapp/webapp/index.html
Hello World
[root@client ~ 11:23:15]# umount /webapp/webapp
umount: /webapp/webapp: not mounted.
[root@client ~ 11:23:33]# rbd snap create images_pool/webapp@snap1
Creating snap: 100% complete...done.
[root@client ~ 11:23:42]# rbd snap ls images_pool/webapp
SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP
     4  snap1  2 GiB             Mon Jun  1 11:23:42 2026
[root@client ~ 11:23:48]# mount /dev/rbd/images_pool/webapp /webapp/webapp
[root@client ~ 11:24:08]# echo Hello my > /webapp/webapp/index.html
[root@client ~ 11:25:08]# cat /webapp/webapp/index.html
Hello my
[root@client ~ 11:25:12]# rbd snap ls images_pool/webapp
SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP
     4  snap1  2 GiB             Mon Jun  1 11:23:42 2026
bash 复制代码 
[root@client ~ 11:25:19]# rm /webapp/webapp/index.html
rm: remove regular file '/webapp/webapp/index.html'? yes
[root@client ~ 11:45:41]# umount /dev/rbd/images_pool/webapp
[root@client ~ 11:45:55]#  mkdir /webapp/webapp-snap1
[root@client ~ 11:46:01]# rbd map images_pool/webapp@snap1
/dev/rbd2
[root@client ~ 11:46:10]#  mount /dev/rbd/images_pool/webapp@snap1 /webapp/webapp-snap1
mount: /webapp/webapp-snap1: WARNING: device write-protected, mounted read-only.
[root@client ~ 11:46:17]#  cat /webapp/webapp-snap1/index.html
cat: /webapp/webapp-snap1/index.html: No such file or directory

删除快照

bash 复制代码 
[root@client ~ 13:36:28]#  umount /dev/rbd/images_pool/webapp@snap1
umount: /dev/rbd/images_pool/webapp@snap1: not mounted.
[root@client ~ 13:36:36]# rbd unmap images_pool/webapp@snap1
[root@client ~ 13:36:43]# rbd snap rm images_pool/webapp@snap1
Removing snap: 100% complete...done.
[root@client ~ 13:36:52]# rbd snap ls images_pool/webapp
bash 复制代码 
[root@client ~ 13:36:59]#  rbd snap create images_pool/webapp@snap1
Creating snap: 100% complete...done.
[root@client ~ 13:37:43]#  rbd snap create images_pool/webapp@snap2
Creating snap: 100% complete...done.
[root@client ~ 13:37:50]#  rbd snap ls images_pool/webapp
SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP
     6  snap1  2 GiB             Mon Jun  1 13:37:54 2026
     7  snap2  2 GiB             Mon Jun  1 13:38:01 2026
[root@client ~ 13:37:58]# rbd snap purge images_pool/webapp
Removing all snapshots: 100% complete...done.
[root@client ~ 13:38:08]#  rbd snap ls images_pool/webapp

rbd克隆

bash 复制代码 
#创建快照
[root@client ~ 13:39:18]# rbd snap create images_pool/webapp@snap1
Creating snap: 100% complete...done.
[root@client ~ 14:07:24]# rbd snap ls images_pool/webapp
SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP
    10  snap1  2 GiB             Mon Jun  1 14:07:24 2026
#保护快照以免删除
[root@client ~ 14:08:06]# rbd snap protect images_pool/webapp@snap1
[root@client ~ 14:09:13]# rbd snap ls images_pool/webapp
SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP
    10  snap1  2 GiB  yes        Mon Jun  1 14:07:24 2026
#使用此快照创建克隆
[root@client ~ 14:09:20]#  rbd clone images_pool/webapp@snap1 images_pool/webapp-clone-1
[root@client ~ 14:10:07]# rbd map images_pool/webapp-clone-1
/dev/rbd2
[root@client ~ 14:10:14]# mkdir /webapp/webapp-clone-1
[root@client ~ 14:10:22]# umount /webapp/webapp
[root@client ~ 14:10:30]# mount /dev/rbd/images_pool/webapp-clone-1 /webapp/webapp-clone-1
[root@client ~ 14:10:46]# df /webapp/webapp-clone-1/
Filesystem     1K-blocks  Used Available Use% Mounted on
/dev/rbd2        2086912 47860   2039052   3% /webapp/webapp-clone-1
#新克隆只支持cow
[root@client ~ 14:10:53]# ceph config get client rbd_clone_copy_on_read
false
#查看基于快照的克隆列表
[root@client ~ 14:11:48]# rbd clone images_pool/webapp@snap1 images_pool/webapp-clone-2
[root@client ~ 14:13:28]#  rbd children images_pool/webapp@snap1
images_pool/webapp-clone-1
images_pool/webapp-clone-2
#扁平化克隆
[root@client ~ 14:13:33]#  rbd flatten images_pool/webapp-clone-1
Image flatten: 100% complete...done.
[root@client ~ 14:14:31]# rbd children images_pool/webapp@snap1
images_pool/webapp-clone-2
[root@client ~ 14:14:37]# rbd ls images_pool
webapp
webapp-clone-1
webapp-clone-2
webapp1
#挂载克隆
[root@client ~ 14:14:46]# rbd map images_pool/webapp-clone-2
/dev/rbd3
[root@client ~ 14:16:25]#  uuidgen
44952a3b-784f-4f9e-b5e7-9f95e8621345
[root@client ~ 14:16:33]# tune2fs -U 44952a3b-784f-4f9e-b5e7-9f95e8621345 /dev/rbd3
tune2fs 1.45.6 (20-Mar-2020)
tune2fs: Bad magic number in super-block while trying to open /dev/rbd3
/dev/rbd3 contains a xfs file system
[root@client ~ 14:16:53]#  xfs_admin -U 44952a3b-784f-4f9e-b5e7-9f95e8621345 /dev/rbd3
Clearing log and setting UUID
writing all SBs
new UUID = 44952a3b-784f-4f9e-b5e7-9f95e8621345
[root@client ~ 14:17:35]# mkdir /webapp/webapp-clone-2
[root@client ~ 14:17:55]#  mount /dev/rbd3 /webapp/webapp-clone-2/
[root@client ~ 14:18:07]# lsblk
NAME        MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
sr0          11:0    1  1024M  0 rom
rbd0        252:0    0     2G  0 disk /webapp/webapp1
rbd1        252:16   0     2G  0 disk
rbd2        252:32   0     2G  0 disk /webapp/webapp-clone-1
rbd3        252:48   0     2G  0 disk /webapp/webapp-clone-2
nvme0n1     259:0    0   200G  0 disk
├─nvme0n1p1 259:1    0     1G  0 part /boot
└─nvme0n1p2 259:2    0   199G  0 part
  ├─cs-root 253:0    0    70G  0 lvm  /
  ├─cs-swap 253:1    0   2.1G  0 lvm  [SWAP]
  └─cs-home 253:2    0 126.9G  0 lvm  /home

创建对象存储域

bash 复制代码 
[root@client ~ 14:18:14]# radosgw-admin realm create --rgw-realm=webapp --default
{
    "id": "a274c682-9a06-484e-8463-2c8e85926bb1",
    "name": "webapp",
    "current_period": "074c704a-490c-40db-a920-71132b0ccd90",
    "epoch": 1
}
[root@client ~ 15:03:44]# radosgw-admin realm list
{
    "default_info": "a274c682-9a06-484e-8463-2c8e85926bb1",
    "realms": [
        "webapp"
    ]
}
[root@client ~ 15:06:20]# radosgw-admin zonegroup create --rgw-realm=webapp --rgw-zonegroup=video --master --default
{
    "id": "c1fae33c-992d-425e-8510-c4045a4fe7f3",
    "name": "video",
    "api_name": "video",
    "is_master": "true",
    "endpoints": [],
    "hostnames": [],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [],
    "placement_targets": [],
    "default_placement": "",
    "realm_id": "a274c682-9a06-484e-8463-2c8e85926bb1",
    "sync_policy": {
        "groups": []
    }
}
[root@client ~ 15:06:53]# radosgw-admin zonegroup list
{
    "default_info": "c1fae33c-992d-425e-8510-c4045a4fe7f3",
    "zonegroups": [
        "video"
    ]
}
[root@client ~ 15:08:17]# radosgw-admin zone create --rgw-realm=webapp --rgw-zonegroup=video --rgw-zone=storage1 --master --default
{
    "id": "08780fac-1023-4a61-a971-7aa002282f69",
    "name": "storage1",
    "domain_root": "storage1.rgw.meta:root",
    "control_pool": "storage1.rgw.control",
    "gc_pool": "storage1.rgw.log:gc",
    "lc_pool": "storage1.rgw.log:lc",
    "log_pool": "storage1.rgw.log",
    "intent_log_pool": "storage1.rgw.log:intent",
    "usage_log_pool": "storage1.rgw.log:usage",
    "roles_pool": "storage1.rgw.meta:roles",
    "reshard_pool": "storage1.rgw.log:reshard",
    "user_keys_pool": "storage1.rgw.meta:users.keys",
    "user_email_pool": "storage1.rgw.meta:users.email",
    "user_swift_pool": "storage1.rgw.meta:users.swift",
    "user_uid_pool": "storage1.rgw.meta:users.uid",
    "otp_pool": "storage1.rgw.otp",
    "system_key": {
        "access_key": "",
        "secret_key": ""
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "storage1.rgw.buckets.index",
                "storage_classes": {
                    "STANDARD": {
                        "data_pool": "storage1.rgw.buckets.data"
                    }
                },
                "data_extra_pool": "storage1.rgw.buckets.non-ec",
                "index_type": 0
            }
        }
    ],
    "realm_id": "a274c682-9a06-484e-8463-2c8e85926bb1",
    "notif_pool": "storage1.rgw.log:notif"
}
[root@client ~ 15:10:25]# radosgw-admin zone list
{
    "default_info": "08780fac-1023-4a61-a971-7aa002282f69",
    "zones": [
        "storage1"
    ]
}
[root@client ~ 15:10:30]# radosgw-admin period update --rgw-realm=webapp --commit
{
    "id": "80cdf291-bc7d-4763-9e84-0b8804d9ed5a",
    "epoch": 1,
    "predecessor_uuid": "074c704a-490c-40db-a920-71132b0ccd90",
    "sync_status": [],
    "period_map": {
        "id": "80cdf291-bc7d-4763-9e84-0b8804d9ed5a",
        "zonegroups": [
            {
                "id": "c1fae33c-992d-425e-8510-c4045a4fe7f3",
                "name": "video",
                "api_name": "video",
                "is_master": "true",
                "endpoints": [],
                "hostnames": [],
                "hostnames_s3website": [],
                "master_zone": "08780fac-1023-4a61-a971-7aa002282f69",
                "zones": [
                    {
                        "id": "08780fac-1023-4a61-a971-7aa002282f69",
                        "name": "storage1",
                        "endpoints": [],
                        "log_meta": "false",
                        "log_data": "false",
                        "bucket_index_max_shards": 11,
                        "read_only": "false",
                        "tier_type": "",
                        "sync_from_all": "true",
                        "sync_from": [],
                        "redirect_zone": ""
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": [],
                        "storage_classes": [
                            "STANDARD"
                        ]
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "a274c682-9a06-484e-8463-2c8e85926bb1",
                "sync_policy": {
                    "groups": []
                }
            }
        ],
        "short_zone_ids": [
            {
                "key": "08780fac-1023-4a61-a971-7aa002282f69",
                "val": 1428229327
            }
        ]
    },
    "master_zonegroup": "c1fae33c-992d-425e-8510-c4045a4fe7f3",
    "master_zone": "08780fac-1023-4a61-a971-7aa002282f69",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        }
    },
    "realm_id": "a274c682-9a06-484e-8463-2c8e85926bb1",
    "realm_name": "webapp",
    "realm_epoch": 2
}
[root@client ~ 15:10:48]# radosgw-admin zone get --rgw-zone=storage1
{
    "id": "08780fac-1023-4a61-a971-7aa002282f69",
    "name": "storage1",
    "domain_root": "storage1.rgw.meta:root",
    "control_pool": "storage1.rgw.control",
    "gc_pool": "storage1.rgw.log:gc",
    "lc_pool": "storage1.rgw.log:lc",
    "log_pool": "storage1.rgw.log",
    "intent_log_pool": "storage1.rgw.log:intent",
    "usage_log_pool": "storage1.rgw.log:usage",
    "roles_pool": "storage1.rgw.meta:roles",
    "reshard_pool": "storage1.rgw.log:reshard",
    "user_keys_pool": "storage1.rgw.meta:users.keys",
    "user_email_pool": "storage1.rgw.meta:users.email",
    "user_swift_pool": "storage1.rgw.meta:users.swift",
    "user_uid_pool": "storage1.rgw.meta:users.uid",
    "otp_pool": "storage1.rgw.otp",
    "system_key": {
        "access_key": "",
        "secret_key": ""
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "storage1.rgw.buckets.index",
                "storage_classes": {
                    "STANDARD": {
                        "data_pool": "storage1.rgw.buckets.data"
                    }
                },
                "data_extra_pool": "storage1.rgw.buckets.non-ec",
                "index_type": 0
            }
        }
    ],
    "realm_id": "",
    "notif_pool": "storage1.rgw.log:notif"
}

rados网关部署

bash 复制代码 
[root@client ~ 15:10:56]# ceph orch apply rgw webapp --placement="3 ceph1.my.cloud ceph2.my.cloud ceph3.my.cloud" --realm=webapp --zone=storage1 --port=8080
Scheduled rgw.webapp update...
[root@client ~ 15:22:31]# ceph orch ls rgw
NAME        PORTS   RUNNING  REFRESHED  AGE  PLACEMENT                                            
rgw.webapp  ?:8080      1/3  -          10s  ceph1.my.cloud;ceph2.my.cloud;ceph3.my.cloud;count:3 
[root@client ~ 15:22:51]# ceph orch ps --daemon-type rgw| awk '{print $1,$4}'
NAME STATUS
rgw.webapp.ceph1.xbzjin running
rgw.webapp.ceph2.rhjcrt running
rgw.webapp.ceph3.pundcm running
#验证
[root@ceph1 ~ 15:30:12]# curl http://ceph1.my.cloud:8080
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>[root@ceph1 ~ 15:30:32]# curl http://ceph2.my.cloud:8080
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>[root@ceph1 ~ 15:30:40]# curl http://ceph3.my.cloud:8080
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>

创建用户

bash 复制代码 
 [root@ceph1 ~ 15:30:45]# radosgw-admin user create --uid="operator" --display-name="langdie" --email="operator@example.com" --access-key="12345" --secret-key="67890"
{
    "user_id": "operator",
    "display_name": "langdie",
    "email": "operator@example.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "operator",
            "access_key": "12345",
            "secret_key": "67890"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

[root@ceph1 ~ 15:42:22]# radosgw-admin user list
[
    "operator",
    "dashboard"
]
[root@ceph1 ~ 15:42:34]# radosgw-admin user info --uid=operator
{
    "user_id": "operator",
    "display_name": "langdie",
    "email": "operator@example.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "operator",
            "access_key": "12345",
            "secret_key": "67890"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

不指定属性则随机生成

bash 复制代码 
[root@ceph1 ~ 15:43:57]# radosgw-admin user create --uid=s3user --display-name="yingwu"
{
    "user_id": "s3user",
    "display_name": "yingwu",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "s3user",
            "access_key": "7B0SM89MASRCWASZEX9E",
            "secret_key": "dBvlum5YpevGJFih7K6fKTvJJDSasj23xPQeFi4l"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

重新生成密钥

bash 复制代码 
[root@ceph1 ~ 15:45:24]# radosgw-admin key create --uid=s3user --access-key="ZQI72JZZDTA8BRCQOLGK" --gen-secret
{
    "user_id": "s3user",
    "display_name": "yingwu",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "s3user",
            "access_key": "7B0SM89MASRCWASZEX9E",
            "secret_key": "dBvlum5YpevGJFih7K6fKTvJJDSasj23xPQeFi4l"
        },
        {
            "user": "s3user",
            "access_key": "ZQI72JZZDTA8BRCQOLGK",
            "secret_key": "oAxtSsDkV3mttTO8TSC2utGto7lJbHq4BkfTff3x"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

添加用户访问密钥

bash 复制代码 
[root@ceph1 ~ 15:45:31]# radosgw-admin key create --uid=s3user --gen-access-key
{
    "user_id": "s3user",
    "display_name": "yingwu",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "s3user",
            "access_key": "7B0SM89MASRCWASZEX9E",
            "secret_key": "dBvlum5YpevGJFih7K6fKTvJJDSasj23xPQeFi4l"
        },
        {
            "user": "s3user",
            "access_key": "QYXDDZ6IK553MIQKZKIV",
            "secret_key": "L6wDeFDa2xx3LzjcVICm4rtIv7L6SVAOOT434Fw2"
        },
        {
            "user": "s3user",
            "access_key": "ZQI72JZZDTA8BRCQOLGK",
            "secret_key": "oAxtSsDkV3mttTO8TSC2utGto7lJbHq4BkfTff3x"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

删除用户密钥

bash 复制代码 
[root@ceph1 ~ 15:47:04]# radosgw-admin key rm --uid=s3user --access-key=7B0SM89MASRCWASZEX9E
{
    "user_id": "s3user",
    "display_name": "yingwu",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "s3user",
            "access_key": "QYXDDZ6IK553MIQKZKIV",
            "secret_key": "L6wDeFDa2xx3LzjcVICm4rtIv7L6SVAOOT434Fw2"
        },
        {
            "user": "s3user",
            "access_key": "ZQI72JZZDTA8BRCQOLGK",
            "secret_key": "oAxtSsDkV3mttTO8TSC2utGto7lJbHq4BkfTff3x"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

删除用户密钥

bash 复制代码 
[root@ceph1 ~ 15:47:34]# radosgw-admin key rm --uid=s3user --access-key=QYXDDZ6IK553MIQKZKIV
{
    "user_id": "s3user",
    "display_name": "yingwu",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "s3user",
            "access_key": "ZQI72JZZDTA8BRCQOLGK",
            "secret_key": "oAxtSsDkV3mttTO8TSC2utGto7lJbHq4BkfTff3x"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

临时禁用对象网关用户

bash 复制代码 
[root@ceph1 ~ 15:49:00]# radosgw-admin user suspend --uid=s3user
{
    "user_id": "s3user",
    "display_name": "yingwu",
    "email": "",
    "suspended": 1,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
。。。

临时禁用对象网关用户

bash 复制代码 
[root@ceph1 ~ 15:49:39]# radosgw-admin user enable --uid=s3user
{
    "user_id": "s3user",
    "display_name": "yingwu",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
。。。

修改用户信息

bash 复制代码 
[root@ceph1 ~ 15:50:24]# radosgw-admin user modify --uid=s3user --display-name=luojiesi
{
    "user_id": "s3user",
    "display_name": "luojiesi",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "s3user",
            "access_key": "ZQI72JZZDTA8BRCQOLGK",
            "secret_key": "oAxtSsDkV3mttTO8TSC2utGto7lJbHq4BkfTff3x"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

[root@ceph1 ~ 15:51:41]# radosgw-admin user modify --uid=s3user --access=full
{
    "user_id": "s3user",
    "display_name": "luojiesi",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "s3user",
            "access_key": "ZQI72JZZDTA8BRCQOLGK",
            "secret_key": "oAxtSsDkV3mttTO8TSC2utGto7lJbHq4BkfTff3x"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

删除用户（同时删对象和桶）

bash 复制代码 
[root@ceph1 ~ 15:51:57]# radosgw-admin user list
[
    "operator",
    "dashboard",
    "s3user"
]
[root@ceph1 ~ 15:52:23]# radosgw-admin user rm --uid=s3user --purge-data
[root@ceph1 ~ 15:52:29]# radosgw-admin user list
[
    "operator",
    "dashboard"
]

安装 Amazon S3 API 客户端

bash 复制代码 
[root@client ~ 16:31:18]# mkdir .pip
[root@client ~ 16:31:21]#  cat > .pip/pip.conf << 'EOF'
> [global]
> index-url = http://mirrors.aliyun.com/pypi/simple/
> [install]
> trusted-host=mirrors.aliyun.com
> EOF
[root@client ~ 16:31:43]# pip3 install awscli
WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead.
Collecting awscli

配置aws cli 凭据

bash 复制代码 
[root@client ~ 16:32:37]#  aws configure
AWS Access Key ID [None]: 12345
AWS Secret Access Key [None]: 67890
Default region name [None]:
Default output format [None]:

创建存储桶

bash 复制代码 
[root@client ~ 16:33:24]# scp 192.168.108.11:/etc/hosts /etc/hosts
The authenticity of host '192.168.108.11 (192.168.108.11)' can't be established.
ECDSA key fingerprint is SHA256:5nUPAe0IfgEk1mP8rsIRsrWNDyd8rjXi6L1OJrSdbRE.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.108.11' (ECDSA) to the list of known hosts.
root@192.168.108.11's password:
hosts                                                                   100%  304   194.0KB/s   00:00
[root@client ~ 16:34:46]# aws --endpoint=http://ceph1.my.cloud:8080 s3 mb s3://webapp
make_bucket: webapp

查看桶清单

bash 复制代码 
[root@client ~ 16:34:57]# aws --endpoint=http://ceph1.my.cloud:8080 s3 ls
2026-06-01 16:34:54 webapp

上传对象到存储桶

bash 复制代码 
[root@client ~ 16:35:42]# echo Hello World > Welcome-pub.html
[root@client ~ 16:36:29]# echo Hello my > Welcome-pri.html
[root@client ~ 16:37:03]#  aws --endpoint=http://ceph1.my.cloud:8080 s3 cp Welcome-pub.html s3://webapp/ --acl=public-read-write
upload: ./Welcome-pub.html to s3://webapp/Welcome-pub.html
[root@client ~ 16:38:03]#  aws s3 cp Welcome-pri.html s3://webapp --endpoint=http://ceph1.my.cloud:8080   upload: ./Welcome-pri.html to s3://webapp/Welcome-pri.html

查看下载桶中对象

bash 复制代码 
[root@client ~ 16:41:56]# ls /tmp/Welcome-pri.html
/tmp/Welcome-pri.html
[root@client ~ 16:42:03]# aws s3 cp s3://webapp /tmp --recursive --endpoint=http://ceph1.my.cloud:8080
download: s3://webapp/Welcome-pri.html to ../tmp/Welcome-pri.html
download: s3://webapp/Welcome-pub.html to ../tmp/Welcome-pub.html
[root@client ~ 16:43:20]# curl http://ceph1.my.cloud:8080/webapp/Welcome-pub.html
Hello World
[root@client ~ 16:44:51]# curl http://ceph1.my.cloud:8080/webapp/Welcome-pri.html
<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code><Message></Message><BucketName>webapp</BucketName><RequestId>tx00000915cefc1938fb314-006a1d46a0-17256-storage1</RequestId><HostId>17256-storage1-video</HostId></Error>[root@client ~ 16:45:20]#

删除对象与桶

bash 复制代码 
[root@client ~ 16:47:57]# aws s3 rm s3://webapp/Welcome-pri.html --endpoint=http://ceph1.my.cloud:8080    delete: s3://webapp/Welcome-pri.html
[root@client ~ 16:48:42]# aws s3 ls s3://webapp --endpoint=http://ceph1.my.cloud:8080
2026-06-01 16:38:02         12 Welcome-pub.html
[root@client ~ 16:48:59]# aws s3 rb s3://webapp --endpoint=http://ceph1.my.cloud:8080
remove_bucket failed: s3://webapp argument of type 'NoneType' is not iterable
[root@client ~ 16:49:12]# aws s3 rm s3://webapp --recursive --include "Welcome-*" --endpoint=http://ceph1.my.cloud:8080
delete: s3://webapp/Welcome-pub.html
[root@client ~ 16:49:45]# aws s3 rb s3://webapp --endpoint=http://ceph1.my.cloud:8080
remove_bucket: webapp
相关推荐
程序员黑豆2 小时前
AI全栈开发之Java：什么是JDK
前端·后端·ai编程
To_OC2 小时前
万字解析《JS语言精粹》之第四章：函数15大核心精髓（JS灵魂核心）
前端·javascript·代码规范
宋拾壹2 小时前
同时添加多个类目
android·开发语言·javascript
IT知识分享2 小时前
从零开发在线简繁转换工具：OpenCC 实战、避坑经验与方案选型
javascript·python
mqcode2 小时前
Vue3 + Element Plus + Vite 企业级后台框架搭建全流程
前端
SL-staff2 小时前
Web 白板技术架构深度解析：从渲染到协作的选型哲学
前端·架构
川冰ICE2 小时前
JavaScript实战④｜天气查询应用，调用API与异步处理
javascript·css·css3
微扬嘴角2 小时前
react篇4--setState、LazyLoad和Hooks
前端·javascript·react.js
杨梦馨2 小时前
万级数据表格卡死？Web Worker 一招搞定
前端·javascript·vue.js
热门推荐
01《置身钉内》原文-可播放阅读02GitHub 镜像站点03【AI】2026 年具身智能模型和世界模型总结04Codex 下载安装指南：Windows 和 macOS 官方版下载052026 AI 编程工具终极实战指南：Cursor vs Claude Code vs Copilot，开发者该怎么选？062026 年 AI 编程工具终极横评：Cursor vs Claude Code vs Copilot vs Windsurf07【踩坑记录 | 第一篇】微软商店无法使用时，如何手动安装 OpenAI Codex？附`.msix`文件系统错误解决方法08AI科技热点日报 | 2026年6月1日09CC-Switch 下载、安装与使用配置指南【2026.5.29】10CC-Switch & Claude 基于 Linux 服务器安装使用指南