KnoAI 技术站

主题

Ceph 分布式存储设计 ​

1. Ceph 集群架构 ​

1.1 组件部署 ​

┌──────────────────────────────────────────────────────┐
│                   Ceph 集群                           │
│                                                      │
│  ┌──────────┐  ┌──────────┐  ┌──────────┐           │
│  │ MON-1    │  │ MON-2    │  │ MON-3    │           │
│  │ MGR-1    │  │ MGR-2    │  │ MGR-3    │           │
│  └──────────┘  └──────────┘  └──────────┘           │
│                                                      │
│  ┌────────────────────────────────────────────────┐  │
│  │                OSD 节点 (x30)                   │  │
│  │  ┌──────────────────────────────────────────┐  │  │
│  │  │ 每节点: 12 x 8TB HDD + 2 x 1.92TB NVMe │  │  │
│  │  │ OSD-1..12 (HDD) + DB/WAL on NVMe        │  │  │
│  │  └──────────────────────────────────────────┘  │  │
│  └────────────────────────────────────────────────┘  │
│                                                      │
│  总容量: 30 × 12 × 8TB = 2,880TB (裸)                │
│  可用: 2,880 / 3 = 960TB (3副本)                     │
│  安全可用: 960 × 75% = 720TB                         │
└──────────────────────────────────────────────────────┘

1.2 网络配置 ​

ini
# /etc/ceph/ceph.conf

[global]
fsid = <uuid>
mon_initial_members = mon1, mon2, mon3
mon_host = 10.0.3.1, 10.0.3.2, 10.0.3.3
public_network = 10.0.3.0/24    # Ceph 客户端通信
cluster_network = 10.0.4.0/24   # Ceph OSD 复制通信
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

[mon]
mon_data = /var/lib/ceph/mon/$cluster-$id

[osd]
# 性能优化
osd_memory_target = 8589934592       # 8GB per OSD
osd_recovery_max_active = 3
osd_max_backfills = 1
osd_recovery_op_priority = 3
osd_recovery_sleep = 0.1
osd_op_threads = 8
osd_recovery_max_active_hdd = 1

# BlueStore 优化
bluestore_cache_size_hdd = 4294967296  # 4GB cache
bluestore_min_alloc_size_hdd = 65536   # 64KB

2. 存储池规划 ​

2.1 创建存储池 ​

bash
# Cinder 块存储池 (三副本)
ceph osd pool create volumes 256 256
ceph osd pool set volumes size 3
ceph osd pool set volumes min_size 2
ceph osd pool set volumes crush_rule replicated_rule_rack

# Nova 实例盘池 (三副本)
ceph osd pool create vms 512 512
ceph osd pool set vms size 3
ceph osd pool set vms min_size 2

# Glance 镜像池 (三副本)
ceph osd pool create images 128 128
ceph osd pool set images size 3
ceph osd pool set images min_size 2

# 备份池 (纠删码 EC, 节省空间)
ceph osd profile create ec-backup \
  k=4 m=2 \
  crush-failure-domain=rack
ceph osd pool create backups 128 128 erasure ec-backup

# 为每个池创建应用标签
ceph osd pool application enable volumes rbd
ceph osd pool application enable vms rbd
ceph osd pool application enable images rbd
ceph osd pool application enable backups rgw

2.2 CRUSH 规则 (跨机架) ​

bash
# 创建跨机架的 CRUSH 规则
ceph osd crush rule create-replicated \
  replicated_rule_rack default rack

# 查看 CRUSH 树
ceph osd crush tree

# 验证机架分布
ceph osd df tree rack

2.3 QoS 策略 ​

bash
# Cinder 池 QoS (防止单租户占用全部 IO)
ceph osd pool set volumes qos_max_bytes 104857600    # 100MB/s
ceph osd pool set volumes qos_max_iops 10000          # 10K IOPS

# Nova 实例盘 QoS
ceph osd pool set vms qos_max_bytes 209715200         # 200MB/s
ceph osd pool set vms qos_max_iops 20000

# 优先级设置
ceph osd pool set vms pg_autoscale_mode on
ceph osd pool set volumes pg_autoscale_mode on

3. Ceph 与 OpenStack 集成 ​

3.1 创建 Ceph 用户 ​

bash
# Cinder 用户
ceph auth get-or-create client.cinder \
  mon 'profile rbd' \
  osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images'

# Glance 用户
ceph auth get-or-create client.glance \
  mon 'profile rbd' \
  osd 'profile rbd pool=images'

# Nova 用户
ceph auth get-or-create client.nova \
  mon 'profile rbd' \
  osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images'

# 导出 keyring
ceph auth get-key client.cinder | tee /etc/ceph/cinder.key
ceph auth get-key client.glance | tee /etc/ceph/glance.key

3.2 Cinder 配置 ​

ini
# /etc/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends = ceph-rbd

[ceph-rbd]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
rbd_secret_uuid = <uuid>
rbd_store_chunk_size = 4
rbd_max_clone_depth = 5
rbd_flatten_volume_from_snapshot = false
report_discard_supported = true

3.3 Nova 配置 (RBD 后端) ​

ini
# /etc/kolla/config/nova/nova-compute.conf
[libvirt]
images_type = rbd
images_rbd_pool = vms
images_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = nova
rbd_secret_uuid = <uuid>
disk_cachemodes = network=writeback
hw_disk_discard = unmap

4. 性能优化 ​

4.1 NVMe WAL/DB 布局 ​

bash
# NVMe 分区方案 (以 1.92TB NVMe 为例)
# /dev/nvme0n1 分区规划:
#
# 分区          大小       用途
# ──────────   ────────   ─────────
# nvme0n1p1    20GB       WAL for OSD-1
# nvme0n1p2    20GB       WAL for OSD-2
# nvme0n1p3    20GB       WAL for OSD-3
# nvme0n1p4    20GB       WAL for OSD-4
# nvme0n1p5    20GB       WAL for OSD-5
# nvme0n1p6    20GB       WAL for OSD-6
# nvme0n1p7    280GB      DB for OSD-1
# nvme0n1p8    280GB      DB for OSD-2
# nvme0n1p9    280GB      DB for OSD-3
# nvme0n1p10   280GB      DB for OSD-4
# nvme0n1p11   280GB      DB for OSD-5
# nvme0n1p12   280GB      DB for OSD-6

4.2 监控与告警 ​

bash
# Ceph Dashboard (启用)
ceph mgr module enable dashboard
ceph dashboard set-ssl-certificate -i /etc/ceph/cert.pem
ceph dashboard set-ssl-certificate-key -i /etc/ceph/key.pem

# Prometheus 导出
ceph mgr module enable prometheus
# 访问: http://<mgr-ip>:9283/metrics

# 告警规则
ceph dashboard set-alertmanager-api-host http://mgmt-01:9093

5. 容量扩展 ​

5.1 添加 OSD 节点 ​

bash
# 1. 在新节点安装 Ceph OSD (通过 Kolla-Ansible)
kolla-ansible -i inventory deploy --tags ceph

# 2. 验证新 OSD 加入集群
ceph osd tree
ceph osd df

# 3. 等待 rebalance 完成
ceph -w  # 观察 HEALTH_OK

# 4. 调整 PG 数 (如果池增大)
ceph osd pool set volumes pg_num 512
ceph osd pool set volumes pgp_num 512

5.2 容量预警阈值 ​

水位阈值操作
🟢 正常< 65%无需操作
🟡 注意65-75%制定扩容计划
🟠 警告75-85%执行扩容，限制写入
🔴 危险> 85%紧急扩容，nearfull 告警
⛔ 临界> 95%full 状态，拒绝写入