使用kubeadm安装k8s v1.31+containerd+外部etcd+haproxy负载均衡

小半杯绿茶2024-12-06 16:26

k8s部署方案

1. 环境设置

1.1 虚拟机环境

  • 远程操作环境: MacOS - bash
  • 虚拟机-环境: Windows 10
  • 虚拟机-平台: Oracle VM VirtualBox
  • 虚拟机-系统: CentOS-Stream-9-latest-x86_64
  • 虚拟机-网卡-1: enp0s3，桥接网卡，网段: 192.168.0.1/24
  • 虚拟机-网卡-2: enp0s8，仅内网通信，网段: 169.0.0.0/8

1.2 服务器环境

  1. 执行initserver自动化脚本

  2. 网卡设置固定的ip

  3. ~/.bash_profile设置代理

  4. 网络环境

    主机 enp0s3 enp0s8 service
    sre-lo-test-vm-master-001 192.168.0.22,192.168.0.28(vip) 169.0.0.100 etcd, proxy, k8s-master
    sre-lo-test-vm-master-002 192.168.0.23 169.0.0.101 etcd, proxy, k8s-master
    sre-lo-test-vm-master-003 192.168.0.24 169.0.0.102 etcd, proxy, k8s-master
    sre-lo-test-vm-node-001 192.168.0.25 169.0.0.103 k8s-node
    sre-lo-test-vm-node-002 192.168.0.26 169.0.0.104 k8s-node
    sre-lo-test-vm-node-003 192.168.0.27 169.0.0.105 k8s-node
    host subnet 169.0.0.0/16
    k8s pod subnet 169.1.0.0/16
    k8s service subnet 169.2.0.0/16

1.3 配置 Mac OS 远程操作环境

shell 复制代码 
echo '# k8s cluster node
192.168.0.22 sre-lo-test-vm-master-001
192.168.0.23 sre-lo-test-vm-master-002
192.168.0.24 sre-lo-test-vm-master-003
192.168.0.25 sre-lo-test-vm-node-001
192.168.0.26 sre-lo-test-vm-node-002
192.168.0.27 sre-lo-test-vm-node-003
' | sudo tee -a /etc/hosts

ssh-keyscan sre-lo-test-vm-master-001 >> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-master-002 >> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-master-003 >> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-node-001 	>> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-node-002 	>> ~/.ssh/known_hosts
ssh-keyscan sre-lo-test-vm-node-003 	>> ~/.ssh/known_hosts

scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-master-001:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-master-002:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-master-003:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-node-001:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-node-002:~/.ssh
scp -i ~/.ssh/id_rsa ~/.ssh/id_rsa root@sre-lo-test-vm-node-003:~/.ssh

1.4 服务器环境初始化（部分）

shell 复制代码 
# ALL
echo '
169.0.0.100 sre-lo-test-vm-master-001 # etcd, proxy, k8s-master
169.0.0.101 sre-lo-test-vm-master-002 # etcd, proxy, k8s-master
169.0.0.102 sre-lo-test-vm-master-003 # etcd, proxy, k8s-master
169.0.0.103 sre-lo-test-vm-node-001 	 # k8s-node
169.0.0.104 sre-lo-test-vm-node-002	 # k8s-node
169.0.0.105 sre-lo-test-vm-node-003   # k8s-node
' | sudo tee -a /etc/hosts

echo '
export proxy_ip="192.168.0.10"
export proxy_port="9527"
export http_proxy="http://${proxy_ip}:${proxy_port}"
export https_proxy="http://${proxy_ip}:${proxy_port}"
export socks_proxy="http://${proxy_ip}:${proxy_port}"
export ftp_proxy="http://${proxy_ip}:${proxy_port}"
export no_proxy=".cluster.local,cluster.local,localhost,127.0.0.1,localaddress,.localdomain.com,192.168.0.0/16,169.0.0.0/8,172.16.0.0/12"
' | tee -a ~/.bash_profile

source ~/.bash_profile

2. Etcd集群部署

2.1 下载软件

2.1.1 下载cfssl安装包
shell 复制代码 
# MacOS
cd ~/Downloads
curl -L -O https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_amd64
curl -L -O https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_amd64
curl -L -O https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl-certinfo_1.6.4_linux_amd64

mv cfssl_1.6.4_linux_amd64 cfssl
mv cfssljson_1.6.4_linux_amd64 cfssljson
mv cfssl-certinfo_1.6.4_linux_amd64 cfssl-certinfo

scp cfssl root@sre-lo-test-vm-master-001:/usr/local/bin/cfssl
scp cfssljson root@sre-lo-test-vm-master-001:/usr/local/bin/cfssljson
scp cfssl-certinfo root@sre-lo-test-vm-master-001:/usr/bin/cfssl-certinfo

# sre-lo-test-vm-master-001
sudo chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/bin/cfssl-certinfo
2.1.2 下载etcd安装包
shell 复制代码 
# MacOS
cd ~/Downloads
curl -L -O https://github.com/etcd-io/etcd/releases/download/v3.5.16/etcd-v3.5.16-linux-amd64.tar.gz
scp etcd-v3.5.16-linux-amd64.tar.gz root@sre-lo-test-vm-master-001:~/
scp etcd-v3.5.16-linux-amd64.tar.gz root@sre-lo-test-vm-master-002:~/
scp etcd-v3.5.16-linux-amd64.tar.gz root@sre-lo-test-vm-master-003:~/

2.2 部署etcd

2.2.1 生成密钥文件
shell 复制代码 
# sre-lo-test-vm-master-001
mkdir -p /tmp/etcd/tls
cd /tmp/etcd/tls

echo '{
    "signing": {
        "default": {
            "expiry": "87600h"
        },
        "profiles": {
            "www": {
                "expiry": "87600h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}' | sudo tee /tmp/etcd/tls/ca-config.json

echo '{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}' | sudo tee /tmp/etcd/tls/ca-csr.json

echo '{
    "CN": "etcd",
    "hosts": [
        "169.0.0.100",
        "169.0.0.101",
        "169.0.0.102"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}' | sudo tee /tmp/etcd/tls/server-csr.json


cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
	-config=ca-config.json -profile=www server-csr.json \
  | cfssljson -bare server
2.2.2 安装etcd应用
shell 复制代码 
# 创建目录 sre-lo-test-vm-master-*
sudo mkdir -p /data/etcd/{bin,cfg,ssl,data}
sudo tar -zxvf ~/etcd-v3.5.16-linux-amd64.tar.gz -C ~/
sudo mv -f ~/etcd-v3.5.16-linux-amd64/{etcd,etcdctl,etcdutl} /data/etcd/bin/

# 复制密钥 sre-lo-test-vm-master-001
sudo /bin/cp -rf /tmp/etcd/tls/ca*pem /data/etcd/ssl/
sudo /bin/cp -rf /tmp/etcd/tls/server*pem /data/etcd/ssl/
scp /data/etcd/ssl/* root@sre-lo-test-vm-master-002:/data/etcd/ssl/
scp /data/etcd/ssl/* root@sre-lo-test-vm-master-003:/data/etcd/ssl/

# sre-lo-test-vm-master-001
echo '#[Member]
ETCD_NAME="sre-lo-test-vm-master-001"
ETCD_DATA_DIR="/data/etcd/data"
ETCD_LISTEN_PEER_URLS="https://169.0.0.100:2380"
ETCD_LISTEN_CLIENT_URLS="https://169.0.0.100:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://169.0.0.100:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://169.0.0.100:2379"
ETCD_INITIAL_CLUSTER="sre-lo-test-vm-master-001=https://169.0.0.100:2380,sre-lo-test-vm-master-002=https://169.0.0.101:2380,sre-lo-test-vm-master-003=https://169.0.0.102:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"' | sudo tee /data/etcd/cfg/etcd.conf

# sre-lo-test-vm-master-002
echo '#[Member]
ETCD_NAME="sre-lo-test-vm-master-002"
ETCD_DATA_DIR="/data/etcd/data"
ETCD_LISTEN_PEER_URLS="https://169.0.0.101:2380"
ETCD_LISTEN_CLIENT_URLS="https://169.0.0.101:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://169.0.0.101:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://169.0.0.101:2379"
ETCD_INITIAL_CLUSTER="sre-lo-test-vm-master-001=https://169.0.0.100:2380,sre-lo-test-vm-master-002=https://169.0.0.101:2380,sre-lo-test-vm-master-003=https://169.0.0.102:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"' | sudo tee /data/etcd/cfg/etcd.conf

# sre-lo-test-vm-master-003
echo '#[Member]
ETCD_NAME="sre-lo-test-vm-master-003"
ETCD_DATA_DIR="/data/etcd/data"
ETCD_LISTEN_PEER_URLS="https://169.0.0.102:2380"
ETCD_LISTEN_CLIENT_URLS="https://169.0.0.102:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://169.0.0.102:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://169.0.0.102:2379"
ETCD_INITIAL_CLUSTER="sre-lo-test-vm-master-001=https://169.0.0.100:2380,sre-lo-test-vm-master-002=https://169.0.0.101:2380,sre-lo-test-vm-master-003=https://169.0.0.102:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"' | sudo tee /data/etcd/cfg/etcd.conf

# sre-lo-test-vm-master-*
echo '[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/data/etcd/cfg/etcd.conf
ExecStart=/data/etcd/bin/etcd \
    --cert-file=/data/etcd/ssl/server.pem \
    --key-file=/data/etcd/ssl/server-key.pem \
    --peer-cert-file=/data/etcd/ssl/server.pem \
    --peer-key-file=/data/etcd/ssl/server-key.pem \
    --trusted-ca-file=/data/etcd/ssl/ca.pem \
    --peer-trusted-ca-file=/data/etcd/ssl/ca.pem \
    --logger=zap
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target' | sudo tee /usr/lib/systemd/system/etcd.service
2.2.4 启动
shell 复制代码 
# 至少两台同时执行，才能正常启动
sudo systemctl enable etcd --now
sudo systemctl status etcd

# 检查状态
sudo ETCDCTL_API=3 /data/etcd/bin/etcdctl --cacert=/data/etcd/ssl/ca.pem --cert=/data/etcd/ssl/server.pem --key=/data/etcd/ssl/server-key.pem --endpoints="https://169.0.0.100:2379,https://169.0.0.101:2379,https://169.0.0.102:2379" endpoint health

2.3 备份

shell 复制代码 
# sre-lo-test-vm-master-001
echo '#!/bin/bash
etcd_backup_path="/data/etcd/backup"
if [[ ! -d $etcd_backup_path ]];then 
    mkdir -p $etcd_backup_path; 
fi
sudo ETCDCTL_API=3 /data/etcd/bin/etcdctl --cacert=/data/etcd/ssl/ca.pem --cert=/data/etcd/ssl/server.pem --key=/data/etcd/ssl/server-key.pem --endpoints="https://169.0.0.100:2379,https://169.0.0.101:2379,https://169.0.0.102:2379" snapshot save $etcd_backup_path/etcd-snapshot.`date +%Y%m%d%H%M%S`.db
' | sudo tee /data/etcd/bin/backup.sh

chmod +x /data/etcd/bin/backup.sh

echo '# etcd data backup
30 3 * * * /data/etcd/bin/backup.sh' |sudo tee -a /var/spool/cron/root

3. 负载均衡

3.1 注意事项

  • 由于先在一台服务器部署kubeadm，再部署另外两台，所以后端服务器也要先设置一台，再设置另外两台

  • 前端端口，如果负载均衡与k8s-master在同一节点上，kubelet监听的6443端口会发生冲突，因此前端端口可以改成16443，监听后端6443端口

3.2 部署haproxy与keepalived

参考文档：https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md#options-for-software-load-balancing

shell 复制代码 
# ALL
sudo dnf install -y haproxy keepalived
sudo mv /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

echo 'global
    log /dev/log local0
    log /dev/log local1 notice
    daemon

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 1
    timeout http-request    10s
    timeout queue           20s
    timeout connect         5s
    timeout client          20s
    timeout server          20s
    timeout http-keep-alive 10s
    timeout check           10s

frontend apiserver
    bind *:16443
    mode tcp
    option tcplog
    default_backend apiserverbackend

backend apiserverbackend
    option httpchk GET /healthz
    http-check expect status 200
    mode tcp
    option ssl-hello-chk
    balance     roundrobin
        server      sre-lo-test-vm-master-001   169.0.0.100:6443 check
        server      sre-lo-test-vm-master-002   169.0.0.101:6443 check
        server      sre-lo-test-vm-master-003   169.0.0.102:6443 check
' |sudo tee /etc/haproxy/haproxy.cfg


# sre-lo-test-vm-master-001
echo 'global_defs {
    router_id LVS_DEVEL
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance VI_1 {
    state MASTER
    interface enp0s3
    virtual_router_id 51
    priority 100
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.0.28
    }
    track_script {
        check_apiserver
    }
}
' |sudo tee /etc/keepalived/keepalived.conf


# sre-lo-test-vm-master-002
echo 'global_defs {
    router_id LVS_DEVEL
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface enp0s3
    virtual_router_id 51
    priority 100
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.0.28
    }
    track_script {
        check_apiserver
    }
}
' |sudo tee /etc/keepalived/keepalived.conf


# sre-lo-test-vm-master-003
echo 'global_defs {
    router_id LVS_DEVEL
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface enp0s3
    virtual_router_id 51
    priority 100
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.0.28
    }
    track_script {
        check_apiserver
    }
}
' |sudo tee /etc/keepalived/keepalived.conf


# sre-lo-test-vm-master-*
echo '#!/bin/sh
errorExit() {
    echo "*** $*" 1>&2
    exit 1
}

curl --silent --max-time 2 --insecure https://localhost:16443/ -o /dev/null || errorExit "Error GET https://localhost:16443/"
if ip addr | grep -q 192.168.0.28; then
    curl --silent --max-time 2 --insecure https://192.168.0.28:16443/ -o /dev/null || errorExit "Error GET https://192.168.0.28:16443/"
fi' |sudo tee /etc/keepalived/check_apiserver.sh

3.3 启动

shell 复制代码 
# sre-lo-test-vm-master-*
sudo systemctl enable haproxy --now
sudo systemctl enable keepalived --now

sudo systemctl status haproxy
sudo systemctl status keepalived

sudo systemctl restart haproxy
sudo systemctl restart keepalived

4. container部署

参考文档：https://kubernetes.io/zh-cn/docs/setup/production-environment/container-runtimes/

4.1 设置系统环境

shell 复制代码 
# ALL k8s master, k8s node 
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

sudo sysctl --system

swapoff -a 
sed -ri 's/.*swap.*/#&/' /etc/fstab 
free -h

4.2 下载containerd

shell 复制代码 
# MacOS
curl -L -O https://github.com/containerd/containerd/releases/download/v1.6.36/containerd-1.6.36-linux-amd64.tar.gz
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-master-001:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-master-002:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-master-003:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-node-001:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-node-002:~/
scp containerd-1.6.36-linux-amd64.tar.gz root@sre-lo-test-vm-node-003:~/

curl -L -O https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
scp containerd.service root@sre-lo-test-vm-master-001:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-master-002:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-master-003:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-node-001:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-node-002:/usr/lib/systemd/system/
scp containerd.service root@sre-lo-test-vm-node-003:/usr/lib/systemd/system/

curl -L -O https://github.com/opencontainers/runc/releases/download/v1.2.1/runc.amd64
scp runc.amd64 root@sre-lo-test-vm-master-001:~/
scp runc.amd64 root@sre-lo-test-vm-master-002:~/
scp runc.amd64 root@sre-lo-test-vm-master-003:~/
scp runc.amd64 root@sre-lo-test-vm-node-001:~/
scp runc.amd64 root@sre-lo-test-vm-node-002:~/
scp runc.amd64 root@sre-lo-test-vm-node-003:~/

curl -L -O https://github.com/containernetworking/plugins/releases/download/v1.6.0/cni-plugins-linux-amd64-v1.6.0.tgz
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-master-001:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-master-002:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-master-003:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-node-001:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-node-002:~/
scp cni-plugins-linux-amd64-v1.6.0.tgz root@sre-lo-test-vm-node-003:~/

4.3 安装containerd

shell 复制代码 
# ALL k8s master, k8s node
sudo dnf remove -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

cd ~

mkdir -p /data/containerd/opt
mkdir -p /data/containerd/data
mkdir -p /opt/cni/bin

tar -xzvf containerd-1.6.36-linux-amd64.tar.gz -C /usr/local

install -m 755 runc.amd64 /usr/local/sbin/runc

tar -xzvf cni-plugins-linux-amd64-v1.6.0.tgz -C /opt/cni/bin

4.4 设置containerd

shell 复制代码 
# ALL k8s master, k8s node

## 创建目录
mkdir /etc/containerd
containerd config default > /etc/containerd/config.toml
## 配置 systemd cgroup 驱动
sed -i 's|SystemdCgroup = false|SystemdCgroup = true|g' /etc/containerd/config.toml
## 修改root目录
sed -i 's|/var/lib/containerd|/data/containerd/data|g' /etc/containerd/config.toml
## 修改依赖项目录
sed -i 's|/opt/containerd|/data/containerd/opt|g' /etc/containerd/config.toml
## 修改基础容器
sed -i 's|registry.k8s.io/pause:3.6|registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10|g' /etc/containerd/config.toml
## 修改镜像源
sed -i '/^\s*\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.mirrors\]/a\      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]\n        endpoint = ["https://dockerpull.org"]\n      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]\n        endpoint = ["registry.aliyuncs.com/google_containers"]' /etc/containerd/config.toml
## 镜像源登录账户，用于第三方需要登录的地址（可选）
sed -i '/^\s*\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.configs\]/a\      [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".auth]\n        username = ""\n        password = ""\n' /etc/containerd/config.toml

## 镜像源、镜像源登录账户，效果如下所示
[plugins."io.containerd.grpc.v1.cri".registry]
  [plugins."io.containerd.grpc.v1.cri".registry.configs]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".auth]
      username = ""
      password = ""
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
      endpoint = ["https://dockerpull.org"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
      endpoint = ["registry.aliyuncs.com/google_containers"]

4.5 启动containerd

shell 复制代码 
# ALL k8s master, k8s node 
systemctl daemon-reload

systemctl enable --now containerd

sudo systemctl restart containerd

sudo systemctl status containerd

sudo systemctl show --property=Environment containerd

5. 安装kubernetes

5.1 下载应用

shell 复制代码 
# ALL k8s master, k8s node
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable --now kubelet

sudo crictl config --set runtime-endpoint=unix:///var/run/containerd/containerd.sock

5.2 下载镜像

shell 复制代码 
# ALL k8s master, k8s node
sudo kubeadm config images list

crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.31.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.11.3
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.15-0
crictl pull dockerpull.org/calico/cni:v3.25.0
crictl pull dockerpull.org/calico/kube-controllers:v3.25.0
crictl pull dockerpull.org/calico/node:v3.25.0

5.3 设置kubeadm配置文件

yaml 复制代码 
# sre-lo-test-vm-master-001
# ~/kubeadm-config.yaml
kind: InitConfiguration
apiVersion: kubeadm.k8s.io/v1beta4
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 8760h0m0s
  usages:
  - signing
  - authentication
certificateKey: 07ef165f6723337d68b0eca1c6a29222a44aecadabbbbb79016cd160a397782c
localAPIEndpoint:
  advertiseAddress: 169.0.0.100
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: sre-lo-test-vm-master-001

---
kind: ClusterConfiguration
apiVersion: kubeadm.k8s.io/v1beta4
kubernetesVersion: v1.31.3
clusterName: k8s-dog
caCertificateValidityPeriod: 87600h0m0s
certificateValidityPeriod: 8760h0m0s
certificatesDir: /etc/kubernetes/pki
encryptionAlgorithm: RSA-2048
dns: {}
proxy: {}
scheduler: {}
apiServer: {}
controllerManager: {}
etcd:
  external:
    endpoints:
        - https://169.0.0.100:2379
        - https://169.0.0.101:2379
        - https://169.0.0.102:2379
    caFile: /etc/kubernetes/pki/etcd/ca.crt
    certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
    keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
controlPlaneEndpoint: 192.168.0.28:16443
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
networking:
  dnsDomain: cluster.local
  podSubnet: 169.1.0.0/16
  serviceSubnet: 169.2.0.0/16

---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd

5.4 拷贝etcd密钥到k8s服务器

shell 复制代码 
# sre-lo-test-vm-master-001
sudo mkdir -p /etc/kubernetes/pki
sudo mkdir -p /etc/kubernetes/pki/etcd

sudo /bin/cp -rf /data/etcd/ssl/ca.pem /etc/kubernetes/pki/etcd/ca.crt
sudo /bin/cp -rf /data/etcd/ssl/server.pem /etc/kubernetes/pki/apiserver-etcd-client.crt
sudo /bin/cp -rf /data/etcd/ssl/server-key.pem /etc/kubernetes/pki/apiserver-etcd-client.key

sudo chown root:root /etc/kubernetes/pki/etcd/ca.crt
sudo chown root:root /etc/kubernetes/pki/apiserver-etcd-client.crt
sudo chown root:root /etc/kubernetes/pki/apiserver-etcd-client.key

5.5 首台服务器开始部署

shell 复制代码 
sudo kubeadm init --config ~/kubeadm-config.yaml --upload-certs
shell 复制代码 
# 补充说明

## 2小时以后添加新的节点
## sre-lo-test-vm-master-001
## 从kubeadm-config.yaml中生成新的certificateKey
kubeadm --config kubeadm-config.yaml init phase upload-certs --upload-certs

## 控制平面 certificateKey
kubeadm init phase upload-certs --upload-certs

## CA证书 caCertHashes
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | \
openssl rsa -pubin -outform der 2>/dev/null | \
sha256sum | \
awk '{print $1}'

## token
kubeadm token create
kubeadm token list

5.6 部署其他节点

shell 复制代码 
# sre-lo-test-vm-master-002 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:
  bootstrapToken:
    token: "abcdef.0123456789abcdef"
    apiServerEndpoint: "192.168.0.28:16443"
    caCertHashes:
    - "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
controlPlane:
  localAPIEndpoint:
    advertiseAddress: 169.0.0.101
    bindPort: 6443
  certificateKey: 07ef165f6723337d68b0eca1c6a29222a44aecadabbbbb79016cd160a397782c
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: sre-lo-test-vm-master-002
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml
    
# sre-lo-test-vm-master-003 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:
  bootstrapToken:
    token: "abcdef.0123456789abcdef"
    apiServerEndpoint: "192.168.0.28:16443"
    caCertHashes:
    - "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
controlPlane:
  localAPIEndpoint:
    advertiseAddress: 169.0.0.102
    bindPort: 6443
  certificateKey: 07ef165f6723337d68b0eca1c6a29222a44aecadabbbbb79016cd160a397782c
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: sre-lo-test-vm-master-003
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml

# sre-lo-test-vm-node-001 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:
  bootstrapToken:
    token: "abcdef.0123456789abcdef"
    apiServerEndpoint: "192.168.0.28:16443"
    caCertHashes:
    - "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: sre-lo-test-vm-node-001
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml

# sre-lo-test-vm-node-002 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:
  bootstrapToken:
    token: "abcdef.0123456789abcdef"
    apiServerEndpoint: "192.168.0.28:16443"
    caCertHashes:
    - "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: sre-lo-test-vm-node-002
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml

# sre-lo-test-vm-node-003 注意修改: caCertHashes
echo 'apiVersion: kubeadm.k8s.io/v1beta4
kind: JoinConfiguration
discovery:
  bootstrapToken:
    token: "abcdef.0123456789abcdef"
    apiServerEndpoint: "192.168.0.28:16443"
    caCertHashes:
    - "sha256:d1dc176b72a417c0130da63a4bc12a7b4ec32f68d960d2d3b7037f0304752ec4"
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: sre-lo-test-vm-node-003
' | tee ~/kubeadm-config.yaml
sudo kubeadm join --config ~/kubeadm-config.yaml

5.7 设置kubectl

shell 复制代码 
# 检查
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubectl get nodes

5.8 安装网络插件

shell 复制代码 
# calico
wget https://docs.projectcalico.org/manifests/calico.yaml

# update CALICO_IPV4POOL_CIDR
CALICO_IPV4POOL_CIDR = pod subnet
# update image registry
sed -i 's|docker.io/calico|dockerpull.org/calico|g' calico.yaml

# apply
kubectl apply -f calico.yaml

kubectl -n kube-system get pods
相关推荐
魏 无羡3 小时前
linux CentOS系统上卸载docker
linux·kubernetes·centos
Karoku0663 小时前
【k8s集群应用】kubeadm1.20高可用部署（3master）
运维·docker·云原生·容器·kubernetes
凌虚5 小时前
Kubernetes APF（API 优先级和公平调度）简介
后端·程序员·kubernetes
alden_ygq7 小时前
etcd网关
服务器·数据库·etcd
张声录17 小时前
【ETCD】ETCD Leader 节点写入数据流程概览
数据库·etcd
探索云原生8 小时前
在 K8S 中创建 Pod 是如何使用到 GPU 的： nvidia device plugin 源码分析
ai·云原生·kubernetes·go·gpu
启明真纳8 小时前
elasticache备份
运维·elasticsearch·云原生·kubernetes
永卿0019 小时前
nginx学习总结(不包含安装过程)
运维·nginx·负载均衡
人类群星闪耀时10 小时前
大模型技术优化负载均衡：AI驱动的智能化运维
运维·人工智能·负载均衡
jwolf210 小时前
基于K8S的微服务：一、服务发现，负载均衡测试（附calico网络问题解决）
微服务·kubernetes·服务发现
热门推荐
01〔AI 绘画〕Stable Diffusion 之 解决绘制多人或面部很小的人物时面部崩坏问题 篇02YOLOv8训练好模型后，追加轮数继续训练、或者提前终止训练，缩减训练轮数03玄机平台应急响应—webshell查杀04机器人编程怎么入门？05HCIA-datacom数通题库和录播视频资料06Docker 夺命连环 15 问07SQL 主从数据库实时备份08校验 GPT-4 真实性的三个经典问题：快速区分 GPT-3.5 与 GPT-4，并提供免费测试网站09(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明10VSCode插件 —— Cody AI （免费AI助手！）