ETCD备份还原 - 技术栈

备份只需要在一个节点上备就可以了，每个节点上的数据是同步的；但是数据恢复是需要在每个主节点上进行

1.查看证书位置

bash 复制代码

#查看etcd证书
[root@k8s-master01 manifests]# cat /etc/kubernetes/manifests/kube-apiserver.yaml  |grep etcd
    - --etcd-cafile=/etc/ssl/etcd/ssl/ca.pem
    - --etcd-certfile=/etc/ssl/etcd/ssl/node-k8s-master01.pem
    - --etcd-keyfile=/etc/ssl/etcd/ssl/node-k8s-master01-key.pem

2.ETCD备份

bash 复制代码

ETCDCTL_API=3 etcdctl snapshot save /root/etcd-snapshot-`date +%Y%m%d%H%m`.db \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/ssl/etcd/ssl/ca.pem \
--cert=/etc/ssl/etcd/ssl/node-k8s-master01.pem \
--key=/etc/ssl/etcd/ssl/node-k8s-master01-key.pem

3.单master,ETCD还原

bash 复制代码

rm -rf /var/lib/etcd.bck

#移除所有etcd服务实例的数据目录
mv /etc/kubernetes/manifests /etc/kubernetes/manifests.bck
mv /var/lib/etcd /var/lib/etcd.bck


#使用20230807-0822.db文件恢复数据到/var/lib/etcd目录
ETCDCTL_API=3 etcdctl snapshot restore /root/etcd-snapshot-202308072008.db \
--data-dir=/var/lib/etcd




#启动kube-apiserver和etcd容器
mv /etc/kubernetes/manifests.bck /etc/kubernetes/manifests


#需要重启kubelet，不然删除和创建Pod异常
systemctl restart kubelet

3.1多个master还原

bash 复制代码

rm -rf /var/lib/etcd.bck

#移除所有etcd服务实例的数据目录
mv /etc/kubernetes/manifests /etc/kubernetes/manifests.bck
mv /var/lib/etcd /var/lib/etcd.bck




# k8s-master01 机器上操作
$ ETCDCTL_API=3 etcdctl snapshot restore /root/etcd-snapshot-202308072008.db \
  --endpoints=192.168.1.220 \
  --name k8s-master01 \
  --initial-cluster "k8s-master01=https://192.168.1.220:2380,k8s-master02=https://192.168.1.221:2380,k8s-master03=https://192.168.1.222:2380" \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls https://192.168.1.220:2380 \
  --data-dir=/var/lib/etcd
  
# k8s-master02 机器上操作
$ ETCDCTL_API=3 etcdctl snapshot restore /root/etcd-snapshot-202308072008.db \
  --endpoints=192.168.1.221 \
  --name k8s-master02 \
  --initial-cluster "k8s-master01=https://192.168.1.220:2380,k8s-master02=https://192.168.1.221:2380,k8s-master03=https://192.168.1.222:2380" \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls https://192.168.1.221:2380 \
  --data-dir=/var/lib/etcd
  
# k8s-master03 机器上操作
$ ETCDCTL_API=3 etcdctl snapshot restore /root/etcd-snapshot-202308072008.db \
  --endpoints=192.168.1.222 \
  --name k8s-master03 \
  --initial-cluster "k8s-master01=https://192.168.1.220:2380,k8s-master02=https://192.168.1.221:2380,k8s-master03=https://192.168.1.222:2380" \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls https://192.168.1.222:2380 \
  --data-dir=/var/lib/etcd



#检查etcd
ETCDCTL_API=3 etcdctl --endpoints="https://192.168.1.220:2379,https://192.168.1.221:2379,https://192.168.1.222:2379" \
--cacert=/etc/ssl/etcd/ssl/ca.pem \
--cert=/etc/ssl/etcd/ssl/node-k8s-master01.pem \
--key=/etc/ssl/etcd/ssl/node-k8s-master01-key.pem \
member list --write-out=table


#检查所有etcd健康状态
ETCDCTL_API=3 etcdctl --cacert=/etc/ssl/etcd/ssl/ca.pem \
--cert=/etc/ssl/etcd/ssl/node-k8s-master01.pem \
--key=/etc/ssl/etcd/ssl/node-k8s-master01-key.pem \
--endpoints="https://192.168.1.220:2379,https://192.168.1.221:2379,https://192.168.1.222:2379" \
 endpoint health -w table



#启动kube-apiserver和etcd容器
mv /etc/kubernetes/manifests.bck /etc/kubernetes/manifests


#需要重启kubelet，不然删除和创建Pod异常
systemctl restart kubelet

4.检查ETCD状态

bash 复制代码

kubectl get cs

kubectl get po

5.定时备份

bash 复制代码

whereis etcdctl


cat > /backups/etcd-bak.sh << 'eof'
#!/bin/bash

ETCDCTL_PATH='/usr/local/bin/etcdctl'
ENDPOINTS='https://127.0.0.1:2379'
ETCD_DATA_DIR="/var/lib/etcd"
BACKUP_DIR="/backups/etcd/etcd-$(date +%Y-%m-%d-%H-%M-%S)"

KEEPBACKUPNUMBER='5'
ETCDBACKUPPERIOD='30'
ETCDBACKUPHOUR=''

ETCDCTL_CERT="/etc/ssl/etcd/ssl/node-k8s-master01.pem"
ETCDCTL_KEY="/etc/ssl/etcd/ssl/node-k8s-master01-key.pem"
ETCDCTL_CA_FILE="/etc/ssl/etcd/ssl/ca.pem"

[ ! -d $BACKUP_DIR ] && mkdir -p $BACKUP_DIR

export ETCDCTL_API=2;$ETCDCTL_PATH backup --data-dir $ETCD_DATA_DIR --backup-dir $BACKUP_DIR

sleep 3

{
export ETCDCTL_API=3;$ETCDCTL_PATH --endpoints="$ENDPOINTS" snapshot save $BACKUP_DIR/snapshot.db \
                                   --cacert="$ETCDCTL_CA_FILE" \
                                   --cert="$ETCDCTL_CERT" \
                                   --key="$ETCDCTL_KEY"
} > /dev/null 

find /backups/etcd -maxdepth 1 -mtime +7 |xargs -i rm -fr {}
eof



chmod +x /backups/etcd-bak.sh
cd /backups/

5.1 设置定时任务

bash 复制代码

[root@k8s-master01 backups]# crontab -l
*/5 * * * * /backups/etcd-bak.sh >/dev/null 2>&1