Kubespray在线部署K8s

1.主机名解析

ruby 复制代码

root@kubespray:~# cat /etc/hosts
127.0.0.1       localhost localhost.localdomain localhost4 localhost4.localdomain4
::1             localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.10.160  k8s-master01
192.168.10.161  k8s-master02
192.168.10.162  k8s-master03
192.168.10.163  k8s-worker01
192.168.10.164  k8s-worker02
192.168.10.165  kubespray

2.Kubespray软件要求

bash 复制代码

# 需要高版本openssl
# 主要Python3.10.x以上

3.下载Kubespray

ruby 复制代码

root@kubespray:~# git clone https://github.com/kubernetes-sigs/kubespray.git
root@kubespray:~# ls
_config.yml               inventory                 remove_node.yml
ansible.cfg               library                   remove-node.yml
CHANGELOG.md              LICENSE                   requirements.txt
cluster.yml               logo                      reset.yml
CNAME                     meta                      roles
code-of-conduct.md        OWNERS                    scale.yml
contrib                   OWNERS_ALIASES            scripts
CONTRIBUTING.md           pipeline.Dockerfile       SECURITY_CONTACTS
Dockerfile                playbooks                 test-infra
docs                      plugins                   tests
extra_playbooks           README.md                 upgrade_cluster.yml
galaxy.yml                recover-control-plane.yml upgrade-cluster.yml
index.html                RELEASE.md                Vagrantfile

# remove_node.yml 移除节点
# reset.yml 重置集群节点
# upgrade_cluster.yml 更新集群
# scale.yml 扩容集群
# cluster.yml 部署集群
# inventory文件夹 部署K8s时变量 集群 主机清单配置等 重点修改

4. 修改相关配置

4.1 安装依赖，拷贝样本目录

ruby 复制代码

# 安装Python依赖
root@kubespray:~# pip3 install -r requirements.txt --break-system-packages
root@kubespray:~# ls inventory
local  sample
root@kubespray:~# cp -rfp inventory/sample inventory/mycluster
root@kubespray:~# ls inventory
local  sample mycluster


# 其他
# 定义一个列表存储IP地址
root@kubespray:~# declare -a IPS=(192.168.1.231 192.168.1.232 192.168.1.233 192.168.1.234 192.168.1.235)
root@kubespray:~# echo ${IPS[*]}
root@kubespray:~# echo ${IPS[@]}
# 通过Python生成主机清单
root@kubespray:~# CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]}

4.2 修改配置

主要修改文件

• inventory/mycluster/group_vars/all/*.yml
• inventory/mycluster/group_vars/k8s-cluster/*.yml

4.3 集群网络

makefile 复制代码

# 修改配置文件 
root@kubespray:~# vi inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
# 选择网络插件，支持 cilium, calico, weave 和 flannel
kube_network_plugin: calico
# 设置 Service 网段
kube_service_addresses: 10.96.0.0/12
# 设置 Pod 网段
kube_pods_subnet: 10.244.0.0/16

其它相关配置文件:

inventory/mycluster/group_vars/k8s_cluster/k8s-net-*.yml。

4.4 运行时修改

ruby 复制代码

# 修改配置文件: inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
root@kubespray:~# vi inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
# 支持 docker, crio 和 containerd，推荐 containerd.
container_manager: containerd
# 是否开启 kata containers
kata_containers_enabled: false

其它相关配置文件:

bash 复制代码

inventory/mycluster/group_vars/all/containerd.yml
inventory/mycluster/group_vars/all/cri-o.yml
inventory/mycluster/group_vars/all/docker.yml

4.5 集群证书

shell 复制代码

# 修改配置文件 inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml

# 是否开启自动更新证书，推荐开启。
auto_renew_certificates: true

4.6 准备机器列表

ini 复制代码

root@kubespray:~# vi inventory/mycluster/inventory.ini
[all]
master233 ansible_host=192.168.1.233
# master2 ansible_host=10.10.10.2
# master3 ansible_host=10.10.10.3
slave234 ansible_host=192.168.1.234
slave235 ansible_host=192.168.1.235
# node3 ansible_host=10.10.10.6
# node4 ansible_host=10.10.10.7
# node5 ansible_host=10.10.10.8
# node6 ansible_host=10.10.10.9
# node7 ansible_host=10.10.10.10

[kube_control_plane]
master233
# master2
# master3

[etcd]
master233
# master2
# master3

[kube_node]
# master1
# master2
# master3
slave234
slave235
# node3
# node4
# node5
# node6
# node7

[calico_rr]

[k8s_cluster:children]
kube_control_plane
kube_node
calico_rr

4.7 调整系统配置

ruby 复制代码

# 开启IP转发
root@kubespray:~# ansible all -i inventory/mycluster/inventory.ini -m shell -a "echo 'net.ipv4.ip_forward=1'|tee -a /etc/sysctl.conf"
root@kubespray:~# ansible all -i inventory/mycluster/inventory.ini -m shell -a "sysctl -p"

# 禁用交换分区
root@kubespray:~# ansible all -i inventory/mycluster/inventory.ini -m shell -a "sed -i '/ swap / s/^/#/' /etc/fstab && swapoff -a"

# 如有其他参数需要调整可以一并使用ansible来进行调整

4.8 开始部署

ini 复制代码

root@kubespray:~# ansible-playbook \
  -i inventory/mycluster/inventory.ini --become --become-user=root \
  --private-key=/root/.ssh/id_ed25519 \
  cluster.yml

4.9 将kubeconfig配置复制到本机

csharp 复制代码

root@kubespray:~# ansible -i '192.168.1.233' -b -m fetch --private-key /root/.ssh/id_ed25519 --user=root -a 'src=/root/.kube/config dest=kubeconfig flat=yes' all
[WARNING]: Unable to parse /root/kubespray/192.168.1.233 as an inventory source
[WARNING]: No inventory was parsed, only implicit localhost is available
[WARNING]: provided hosts list is empty, only localhost is available. Note that the
implicit localhost does not match 'all'

获取到 kubeconfig 后，可以修改其中的 server 地址，将 https://127.0.0.1:6443 改为非 master 节点可以访问的地址，最简单就直接替换 127.0.0.1 成其中一台 master 节点的 IP 地址，也可以在 Master 前面挂个负载均衡器，然后替换成负载均衡器的地址。

4.10 扩容节点

如果要扩容节点，可以准备好节点的内网 IP 列表，并追加到之前的 inventory 文件里，然后再次使用 ansible-playbook 运行一次，有点不同的是: cluster.yml 换成 scale.yml:

ini 复制代码

root@kubespray:~# ansible-playbook \
  -i inventory/mycluster/inventory.ini \
  --private-key=id_rsa \
  --user=ubuntu -b \
  scale.yml

4.11 缩容节点

如果有节点不再需要了，我们可以将其移除集群，通常步骤是:

1. kubectl cordon NODE 驱逐节点，确保节点上的服务飘到其它节点上去，参考安全维护或下线节点。
1. 停止节点上的一些 k8s 组件 (kubelet, kube-proxy) 等。
1. kubectl delete NODE 将节点移出集群。
1. 如果节点是虚拟机，并且不需要了，可以直接销毁掉。
  前 3 个步骤，也可以用 kubespray 提供的 remove-node.yml 这个 playbook 来一步到位实现:

ini 复制代码

root@kubespray:~# ansible-playbook \
  -i inventory/mycluster/inventory.ini \
  --private-key=id_rsa \
  --user=ubuntu -b \
  --extra-vars "node=node1,node2" \
  remove-node.yml

--extra-vars 里写要移出的节点名列表，如果节点已经卡死，无法通过 SSH 登录，可以在 --extra-vars 加个 reset_nodes=false 的选项，跳过第二个步骤

4.9 其他可调整参数

vbnet 复制代码

inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml # 集群的配置文件

1.kube代理模式
# Kube-proxy proxyMode configuration.
# Can be ipvs, iptables, nftables
# TODO: it needs to be changed to nftables when the upstream use nftables as default
kube_proxy_mode: ipvs

2.如果使用了负载均衡器mlb或者kubevip，这里需要调整为true
# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface
# must be set to true for MetalLB, kube-vip(ARP enabled) to work
kube_proxy_strict_arp: false

3.集群域名
# Kubernetes cluster name, also will be used as DNS domain
cluster_name: cluster.local

4.DNS缓存是否开启，开启后会加快域名访问，但是可能会造成缓存给与错误回复
# Set manual server if using a custom cluster DNS server
# manual_dns_server: 10.x.x.x
# Enable nodelocal dns cache
enable_nodelocaldns: true
enable_nodelocaldns_secondary: false
nodelocaldns_ip: 169.254.25.10
nodelocaldns_health_port: 9254
nodelocaldns_second_health_port: 9256
nodelocaldns_bind_metrics_host_ip: false
nodelocaldns_secondary_skew_seconds: 5

inventory/mycluster/group_vars/k8s_cluster/addons.yml # K8s插件文件(开启各种功能)
1.安装Helm
# Helm deployment
helm_enabled: false

2.是否部署容器仓库
# Registry deployment
registry_enabled: false
# registry_namespace: kube-system # 空间名称
# registry_storage_class: "" # 存储类叫什么
# registry_disk_size: "10Gi" # 磁盘大小

3.开启资源监控，暴露指标，如果改为true需要将其他注释注销
# Metrics Server deployment
metrics_server_enabled: false
# metrics_server_container_port: 10250
# metrics_server_kubelet_insecure_tls: true
# metrics_server_metric_resolution: 15s
# metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname"
# metrics_server_host_network: false
# metrics_server_replicas: 1

4.网关配置(下面还有很多)
# Gateway API CRDs
gateway_api_enabled: false
# Nginx ingress controller deployment
ingress_nginx_enabled: false

5.如果是云环境打开
# ALB ingress controller deployment
ingress_alb_enabled: false

6.证书管理器
# Cert manager deployment
cert_manager_enabled: false

7.是否部署负载均衡器
# MetalLB deployment
metallb_enabled: false
metallb_speaker_enabled: "{{ metallb_enabled }}"
metallb_namespace: "metallb-system"

8.k8s cicd工具
argocd_enabled: false
# argocd_namespace: argocd