三个 etcd 节点共同操作
下载 etcd(在三个节点都执行)
bash
ETCD_VER=v3.5.6
curl -L https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz \
-o etcd.tar.gz
tar xzf etcd.tar.gz
cp etcd-${ETCD_VER}-linux-amd64/{etcd,etcdctl,etcdutl} /usr/local/bin/
rm -rf etcd-${ETCD_VER}-linux-amd64 etcd.tar.gz
1.2 生成 etcd 证书
在任意一个节点(比如 etcd-1)上使用 `cfssl` 生成证书,然后分发到其他节点。
bash
mkdir -p /etc/etcd/pki && cd /etc/etcd/pki
创建 CA 配置文件:
bash
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"etcd": {
"usages": ["signing", "key encipherment", "server auth", "client auth"],
"expiry": "87600h"
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "etcd-ca",
"key": {
"algo": "rsa",
"size": 2048
}
}
EOF
生成 CA 证书:
bash
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
创建 etcd 证书配置:
bash
cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.220.111",
"192.168.220.112",
"192.168.220.113"
],
"key": {
"algo": "rsa",
"size": 2048
}
}
EOF
生成 etcd 服务端和 peer 证书:
bash
cfssl gencert \
-ca=ca.pem -ca-key=ca-key.pem \
-config=ca-config.json -profile=etcd \
etcd-csr.json | cfssljson -bare etcd
# 为 apiserver 访问 etcd 生成客户端证书
cfssl gencert \
-ca=ca.pem -ca-key=ca-key.pem \
-config=ca-config.json -profile=etcd \
-cn="apiserver-etcd-client" \
etcd-csr.json | cfssljson -bare apiserver-etcd-client
分发证书到三个 etcd 节点:
bash
for host in 192.168.220.111 192.168.220.112 192.168.220.113; do
ssh root@${host} "mkdir -p /etc/etcd/pki"
scp ca.pem etcd.pem etcd-key.pem root@${host}:/etc/etcd/pki/
done
配置 etcd systemd 服务
三个节点分别创建对应的 service 文件:
etcd-1 (192.168.220.111):
bash
cat > /etc/systemd/system/etcd.service <<EOF
[Unit]
Description=etcd
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd \\
--name etcd-1 \\
--data-dir /var/lib/etcd \\
--listen-client-urls https://192.168.220.111:2379,https://127.0.0.1:2379 \\
--advertise-client-urls https://192.168.220.111:2379 \\
--listen-peer-urls https://192.168.220.111:2380 \\
--initial-advertise-peer-urls https://192.168.220.111:2380 \\
--initial-cluster "etcd-1=https://192.168.220.111:2380,etcd-2=https://192.168.220.112:2380,etcd-3=https://192.168.220.113:2380" \\
--initial-cluster-token etcd-cluster \\
--initial-cluster-state new \\
--client-cert-auth \\
--trusted-ca-file /etc/etcd/pki/ca.pem \\
--cert-file /etc/etcd/pki/etcd.pem \\
--key-file /etc/etcd/pki/etcd-key.pem \\
--peer-client-cert-auth \\
--peer-trusted-ca-file /etc/etcd/pki/ca.pem \\
--peer-cert-file /etc/etcd/pki/etcd.pem \\
--peer-key-file /etc/etcd/pki/etcd-key.pem
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
etcd-2 (192.168.220.112):
bash
cat > /etc/systemd/system/etcd.service <<EOF
[Unit]
Description=etcd
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd \\
--name etcd-2 \\
--data-dir /var/lib/etcd \\
--listen-client-urls https://192.168.220.112:2379,https://127.0.0.1:2379 \\
--advertise-client-urls https://192.168.220.112:2379 \\
--listen-peer-urls https://192.168.220.112:2380 \\
--initial-advertise-peer-urls https://192.168.220.112:2380 \\
--initial-cluster "etcd-1=https://192.168.220.111:2380,etcd-2=https://192.168.220.112:2380,etcd-3=https://192.168.220.113:2380" \\
--initial-cluster-token etcd-cluster \\
--initial-cluster-state new \\
--client-cert-auth \\
--trusted-ca-file /etc/etcd/pki/ca.pem \\
--cert-file /etc/etcd/pki/etcd.pem \\
--key-file /etc/etcd/pki/etcd-key.pem \\
--peer-client-cert-auth \\
--peer-trusted-ca-file /etc/etcd/pki/ca.pem \\
--peer-cert-file /etc/etcd/pki/etcd.pem \\
--peer-key-file /etc/etcd/pki/etcd-key.pem
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
etcd-3 (192.168.220.113):
bash
cat > /etc/systemd/system/etcd.service <<EOF
[Unit]
Description=etcd
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd \\
--name etcd-3 \\
--data-dir /var/lib/etcd \\
--listen-client-urls https://192.168.220.113:2379,https://127.0.0.1:2379 \\
--advertise-client-urls https://192.168.220.113:2379 \\
--listen-peer-urls https://192.168.220.113:2380 \\
--initial-advertise-peer-urls https://192.168.220.113:2380 \\
--initial-cluster "etcd-1=https://192.168.220.111:2380,etcd-2=https://192.168.220.112:2380,etcd-3=https://192.168.220.113:2380" \\
--initial-cluster-token etcd-cluster \\
--initial-cluster-state new \\
--client-cert-auth \\
--trusted-ca-file /etc/etcd/pki/ca.pem \\
--cert-file /etc/etcd/pki/etcd.pem \\
--key-file /etc/etcd/pki/etcd-key.pem \\
--peer-client-cert-auth \\
--peer-trusted-ca-file /etc/etcd/pki/ca.pem \\
--peer-cert-file /etc/etcd/pki/etcd.pem \\
--peer-key-file /etc/etcd/pki/etcd-key.pem
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
启动 etcd 并验证:
bash
# 三个节点同时启动
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
在任意一个节点验证集群状态:
预期输出三个节点都是 `started`,有一个 `leader`。
bash
etcdctl --endpoints=https://192.168.220.111:2379,https://192.168.220.112:2379,https://192.168.220.113:2379 \
--cacert=/etc/etcd/pki/ca.pem \
--cert=/etc/etcd/pki/etcd.pem \
--key=/etc/etcd/pki/etcd-key.pem \
member list --write-out=table
bash
etcdctl --endpoints=https://192.168.220.111:2379,https://192.168.220.112:2379,https://192.168.220.113:2379 \
--cacert=/etc/etcd/pki/ca.pem \
--cert=/etc/etcd/pki/etcd.pem \
--key=/etc/etcd/pki/etcd-key.pem \
endpoint status --write-out=table
第二步:部署 Master 节点(192.168.220.110)
2.1 分发 etcd 证书到 Master
bash
mkdir -p /etc/kubernetes/pki/etcd
scp etcd-1:/etc/etcd/pki/ca.pem /etc/kubernetes/pki/etcd/ca.crt
scp etcd-1:/etc/etcd/pki/apiserver-etcd-client.pem /etc/kubernetes/pki/apiserver-etcd-client.crt
scp etcd-1:/etc/etcd/pki/apiserver-etcd-client-key.pem /etc/kubernetes/pki/apiserver-etcd-client.key
2.2 创建 kubeadm 配置文件
bash
# /root/kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v1.23.17
controlPlaneEndpoint: "192.168.220.110:6443"
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
networking:
serviceSubnet: "10.96.0.0/16"
podSubnet: "172.16.0.0/16"
etcd:
external:
endpoints:
- https://192.168.220.111:2379
- https://192.168.220.112:2379
- https://192.168.220.113:2379
caFile: /etc/kubernetes/pki/etcd/ca.crt
certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock