基于k8s高可用web集群

一、构建harbor镜像仓库

1.部署docker本地仓库

[root@harbor ~]# vim /etc/yum.repos.d/docker.repo  #部署yum源 
[docker]
name = docker
baseurl = https://mirrors.aliyun.com/docker-ce/linux/rhel/9.6/x86_64/stable/ 
gpgcheck = 0
​
[root@harbor ~]# dnf install httpd createrepo-y 
#httpd：Apache网页服务，当本地yum源的web服务器
#createrepo：创建yum仓库的工具，把rpm打包成repodata文件，一个合法的yum软件仓库
​
[root@harbor ~]# mkdir  /var/www/html/docker/ -p    
[root@harbor ~]# vim /etc/httpd/conf/httpd.conf     #改端口，避免冲突
Listen 4444
[root@harbor ~]# systemctl enable --now httpd
​
[root@harbor ~]# dnf  install docker-ce --downloadonly  --destdir /mnt/ -y
#--downloadonly:只下载不安装
#--destdir:下载到指定目的目录
​
[root@harbor ~]# mv /mnt/*.rpm /var/www/html/docker/
[root@harbor ~]# createrepo -v /var/www/html/docker/    #生成repodata
​
[root@harbor ~]# vim /etc/yum.repos.d/docker.repo #部署yum源为本地ip
[docker]
name = docker
baseurl = http://172.25.254.254:4444/docker
gpgcheck = 0
​
​
[root@harbor ~]# dnf install docker-ce -y   #测试可以是否用本地yum源下载docker

2.安装docke在harbor仓库节点，并且配置netfilter内核参数

[root@harbor ~]# echo br_netfilter > /etc/modules-load.d/docker_mod.conf
#把网桥内核模块，写到开机自启动加载模块目录底下
​
[root@harbor ~]# modprobe -a br_netfilter #手动加载模块，立刻生效
​
[root@harbor ~]# vim /etc/sysctl.d/docker.conf #编写网桥内核参数
net.bridge.bridge-nf-call-iptables = 1  #把网桥ipv4流量转发给iptables处理
net.bridge.bridge-nf-call-ip6tables = 1 #把ipv6
net.ipv4.ip_forward = 1 #开启三层ipv4转发
​
[root@harbor ~]# systemctl restart  systemd-modules-load.service #重启加载内核模块服务
[root@harbor ~]# sysctl  --system #手动生效内核参数，立即生效
​
[root@harbor ~]# vim /lib/systemd/system/docker.service #编写docker服务启动单元文件
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --iptables=true #加上iptables防火墙规则
​
ps：lib底下为原始单元文件，当docker服务升级时，我们自定义参数则会被覆盖，所以我们可以把lib底下原始单元启动文件复制到/etc/systemd/system/应用.service ，优先级更高不会被包升级覆盖
​
[root@harbor ~]# systemctl daemon-reload    #重新读取服务配置
[root@harbor ~]# systemctl enable --now docker  #开机自启动

知识点：
iptables的四表五链   链相当于检查站，表相当于工具箱
五链：
PREROUTING  外部流量进入内核，在路由判断之前，做DNAT
#DNAT：目标地址转换，外部流量要访问内部容器，把目标宿主机IP和端口转换成内部容器服务端口和IP
​
INPUT：访问宿主机本机流量 
​
FORWARD：需要宿主机转发
​
OUTPUT：宿主机本机产生的进程流量
​
POSTROUTING：从网卡出去的流量，做SNAT
#SNAT:做源地址转换，内部容器流量访问外网，会把虚拟容器IP和端口转换为宿主机的IP及端口
​
四表：
nat表，地址转换表
PREROUTING:DNAT
POSTROUTING:SNAT
OUTPUT
​
filter表 放行，过滤，丢弃数控v包
INPUT FORWARD OUTPUT
​
mangle表 修改数据包TTL，k8s会打标记
五链都有
​
raw表 关闭连接追踪 ，几乎不用

3.生成key

[root@harbor ~]# mkdir /data/certs -p  #创建存放私钥和公钥文件的目录
[root@harbor ~]# openssl req -newkey rsa:4096 \ 
#生成自签名正式 rsa:4096 密钥长度4096位
> -nodes -sha256 -keyout /data/certs/foldwin.org.key \
#-nodes：私钥不设置密码，自签名证书必加
#-sha256：哈希摘要算法
#-keyout：私钥文件生成路径
> -addext "subjectAltName = DNS:reg.foldwin.org" \
#SAN扩展：指定证书绑定的域名，现代浏览器，docker，kubelet校验强制校验SAN,SAN也可以指定多个域名
> -x509 -days 365 -out /data/certs/foldwin.org.crt
#-x509：直接生成自签名证书，不生成证书请求csr文件
#-days 365 证书有效期
#-out：公钥文件生成路径
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN    #国家
State or Province Name (full name) []:ShangHai #省份
Locality Name (eg, city) [Default City]:HuangPu #城市
Organization Name (eg, company) [Default Company Ltd]:K8s #组织
Organizational Unit Name (eg, section) []:Harbor #部门
Common Name (eg, your name or your server's hostname) []:reg.foldwin.org #旧版用来证书绑定域名的。现在用SAN扩展，填同一个域名，保证不出错就行
Email Address []:admin@foldwin.org #邮件

4.编辑harbor配置文件

[root@harbor ~]# tar zxf  harbor-offline-installer-v2.5.4.tgz -C /opt/
[root@harbor ~]# cd /opt/harbor/
[root@harbor harbor]# ls
common.sh  harbor.v2.5.4.tar.gz  harbor.yml.tmpl  install.sh  LICENSE  prepare
[root@harbor harbor]# cp harbor.yml.tmpl harbor.yml 
#harbor.yml.tmpl:配置模板文件
[root@harbor harbor]# vim harbor.yml
hostname: reg.foldwin.org       #harbor访问的域名，必须和SAN扩展域名一样
  certificate: /data/certs/foldwin.org.crt  #公钥路径
  private_key: /data/certs/foldwin.org.key  #私钥路径
harbor_admin_password: 123  #harbor管理员admin密码
​
​
[root@harbor harbor]# ./install.sh --with-chartmuseum
#开启 helm chart仓库组件，会多出chartmuseum容器，可以用来存储helm包，就是相当于说明书，有很多模板文件
​
#编写启动脚本
[root@harbor ~]# vim /etc/systemd/system/harbor.service
[Unit]
Description=harbor with Docker Compose  
Documentation=https://reg.foldwin.org/compose/
After=docker.service network-online.target  #docker、网络就绪之后再启动harbor
Requires=docker.service     #依赖docker服务，docker没启动harbor就无法启动
​
[Service]
Type=oneshot    #oneshot：执行一次任务就退出，不会常驻进程
#可以比作harbor.service就是一个遥控器，按开机键，就把docker这个电视机打开，harbor就结束了，但harbor里的各个容器就像是电视节目，依然播放着
WorkingDirectory=/opt/harbor #执行docker compose命令必须在harbor目录，读取docker-compose.yml
ExecStart=/usr/bin/docker compose up -d #就是systemctl开启服务指令相当于docker compose up -d ---后台启动所有容器
ExecStop=/usr/bin/docker compose down   #这就是相当于停止并删除所有容器
RemainAfterExit=yes #oneshot专用，执行完ExecStart之后标记服务为active，否则直接变成inactive
​
[Install]
WantedBy=multi-user.target  #开机在多用户模式下启动 
#多用户模式：就是允许多个用户登录服务器，可以ssh远程登录
​
[root@harbor harbor]# docker compose  down  #停止harbor，删除所有harbor容器
WARN[0000] /opt/harbor/docker-compose.yml: the attribute `version` is obsolete, it will be ignored, please remove it to avoid potential confusion
[+] down 4/6
 ⠇ Container registryctl       Stopping                                                1.9s
 ✔ Container nginx             Removed                                                 0.2s
 ✔ Container chartmuseum       Removed                                                 0.2s
 ✔ Container harbor-jobservice Removed                                                 0.1s
 ✔ Container harbor-portal     Removed                                                 0.1s
 ⠦ Container harbor-core       Stopping                                                1.7s
WARN[0000] /opt/harbor/docker-compose.yml: the attribute `version` is obsolete, it will be ignored, please remove it to avoid potential confusion
[+] down 4/6
 ⠇ Container registryctl       Stopping                                                1.9s
 ✔ Container nginx             Removed                                                 0.2s
 ✔ Container chartmuseum       Removed                                                 0.2s
 ✔ Container harbor-jobservice Removed                                                 0.1s
 ✔ Container harbor-portal     Removed                                                 0.1s
 ⠦ Container harbor-core       Stopping                                                1.7s
 
 
[root@harbor ~]# systemctl enable --now harbor #实行开机自启动
Created symlink /etc/systemd/system/multi-user.target.wants/harbor.service → /etc/systemd/system/harbor.service.
​
[root@harbor harbor]# docker compose  ps    #查看容器状态

5.启动并验证

[root@harbor harbor]# mkdir  /etc/docker/certs.d/reg.foldwin.org/ -p
[root@harbor harbor]# cp /data/certs/foldwin.org.crt  /etc/docker/certs.d/reg.foldwin.org/ca.crt
#/etc/docker/certs.d ：Docker专属自签名证书信任目录，固定路径
​
[root@harbor harbor]# vim /etc/hosts #本地域名解析
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
172.25.254.24     harbor reg.foldwin.org
​
​
[root@harbor harbor]# systemctl restart docker
[root@harbor harbor]# docker compose up -d
​
[root@harbor harbor]# docker login  reg.foldwin.org -u admin    #登录harbor
Password:
​
WARNING! Your credentials are stored unencrypted in '/root/.docker/config.json'.
Configure a credential helper to remove this warning. See
https://docs.docker.com/go/credential-store/
​
Login Succeeded
​
#出现问题
把docker专属自签名证书目录写成/etc/docker/cert.d 导致认证失败
#纠正步骤
停止容器
先修改目录名称，或者直接删掉重新创建
然后再harbor目录底下使用./prepare自动生成正确的证书文件
再重新启动docker

二、构建部署k8s所需主机

1.主机部署

主机名	ip	配置	角色
harbor	172.25.254.24	cpu1、mem 1G	harbor仓库
master	172.25.254.25	cpu4、mem>4G	master，k8s集群控制节点
node1	172.25.254.26	cpu2、mem 2G	worker，k8s集群工作节点
node2	172.25.254.27	cpu2、mem 2G	worker，k8s集群工作节点

2.所有主机配置

#关闭swap
#k8s要求必须关闭swap内存交换分区，因为swap开启会影响kubelet调度、内存OOM判断，节点必须彻底禁用swap
systemctl disable --now  swap.target #临时关闭swap，取消开机自启动
systemctl mask swap.target  #锁定，做软链接指向/dev/null
sed '/swap/s/^/#/g' -i /etc/fstab   #直接修改把有swap的行注释掉
/swap/s/^/#/g  匹配swap字符串的行，再行首插入#
[root@master ~]# swapoff -a  临时关闭所有swap
[root@master ~]# free -h    #查看swap剩余空间
               total        used        free      shared  buff/cache   available
Mem:           3.5Gi       483Mi       3.1Gi       9.0Mi       231Mi       3.1Gi
Swap:             0B          0B          0B
[root@master ~]# cat /proc/swaps #查看内核加载了哪些swap设备

三、利用Ansible部署k8s运行环境

1.安装ansible

[root@harbor ~]# vim /etc/yum.repos.d/epel.repo #配置ansible的镜像源
[epel]
name = epel
baseurl = https://mirrors.aliyun.com/epel/9/Everything/x86_64/
gpgcheck = 0
​
[root@harbor ~]# dnf install ansible-core.x86_64 ansible.noarch -y #核心资源包和模块插件包

2.配置所需用户

[root@harbor ~]# useradd  devops
[root@harbor ~]# echo 123 | passwd --stdin devops
更改用户 devops 的密码 。
passwd：所有的身份验证令牌已经成功更新

[root@harbor ~]# su - devops
[devops@harbor ~]$ mkdir  ansible
[devops@harbor ~]$ cd ansible/
[devops@harbor ansible]$
[devops@harbor ansible]$ ssh-keygen -t ed25519	#全部空格跳过实现免密登录
[devops@harbor ansible]$ for i in 25 26 27	#把公钥发给其他主机
> do
> ssh-copy-id root@172.25.254.$i
> done

3.生成Ansible的临时配置文件并建立运行环境

[devops@harbor ansible]$ vim inventory #主机清单文件，就是管理的节点IP
[servers]
172.25.254.25
172.25.254.26
172.25.254.27

[devops@harbor ansible]$ vim ansible.cfg 
[defaults]
inventory = /home/devops/ansible/inventory	#指定主机清单文件路径
remote_user = root	#远程被管理的用户
ask_pass = false	#是否交互式询问ssh密码，我们已经使用ssh免密登录所以false
host_key_checking = false	#关闭ssh主机指纹检查
#主机指纹检查，就是你第一次访问主机所出现的提示语

[devops@harbor ansible]$ ansible all -m ping
#all 就是指主机清单里的所有
#-m ping 调用ping模块，ansible里这个不是icmp网络ping，是ssh联通测试
172.25.254.25 | SUCCESS => {
    "ansible_facts": {
        "discovered_interpreter_python": "/usr/bin/python3"
    },
    "changed": false,
    "ping": "pong"
}
172.25.254.26 | SUCCESS => {
    "ansible_facts": {
        "discovered_interpreter_python": "/usr/bin/python3"
    },
    "changed": false,
    "ping": "pong"
}
172.25.254.27 | SUCCESS => {
    "ansible_facts": {
        "discovered_interpreter_python": "/usr/bin/python3"
    },
    "changed": false,
    "ping": "pong"
}


#建立用户
[devops@harbor ansible]$ ansible all -m user -a "name=devops state=present"
#-a 给模块传递参数
#name：用户名
#state=present 确保用户存在，不存在就创建，已存在就不会重复创建
[devops@harbor ansible]$ ansible all -m shell -a 'echo "123" | passwd --stdin devops'
[devops@harbor ansible]$ ansible all -m lineinfile -a 'path=/etc/sudoers line="devops    ALL=(ALL)    NOPASSWD: ALL" insertafter=EOF'
#lineinfile行编辑模块，专门用来修改文本文件，重复执行不会重复写很多行
#sudoers sudo权限配置文件
#后面line参数是 = 指定用户	可以在所有终端访问所有用户 不用密码 插入到文件最后一行

4.生成ansible的主配置文件并测试

[devops@harbor ansible]$ vim ansible.cfg
[defaults]
inventory = /home/devops/ansible/inventory
remote_user = devops
ask_pass = false
host_key_checking = false

[privilege_escalation]
become = true
become_method = sudo
become_user = root
become_ask_pass = false

#把公钥给devops用户
[devops@harbor ansible]$ for i in 25 26 27 
> do
> ssh-copy-id devops@172.25.254.$i
> done

#测试，发现是root用户，说明sudo成功
[devops@harbor ansible]$ ansible all -m shell -a 'whoami'
172.25.254.27 | CHANGED | rc=0 >>
root
172.25.254.25 | CHANGED | rc=0 >>
root
172.25.254.26 | CHANGED | rc=0 >>
root

5.利用ansible剧本部署docker

[devops@harbor ansible]$ vim k8s-devops.yml
- name: devops k8s from ansible
  hosts: all	#主机名单
  tasks:	#任务列表
  - name: setup docker repo
    yum_repository:	ansible的模块。在远端节点创建yum的.repo源文件
      name: docker	#仓库名称
      description: docker	#描述仓库信息
      baseurl: http://172.25.254.24:4444/docker	#yum源地址
      file: docker	#repo文件名
      gpgcheck: no	#关闭gpg签名校验

  - name: install docker
    dnf:	#ansible dnf软件包管理模块
      name: docker-ce	#要安装的软件包名称
      state: present	#已经安装则不会重复安装

  - name: copy docker.service
    shell: cp /lib/systemd/system/docker.service	#shell模块 远端执行shell命令 /etc/systemd/system/docker.service
    args:	#给shell模块传递参数
      creates: /etc/systemd/system/docker.service	#目标文件存在的话就跳过这条shell命令，避免每次执行都复制

  - name: setup docker.service
    replace:	#ansible替换文件内容模块，对文件做正则替换
      path: /etc/systemd/system/docker.service	#要修改的远端文件路径
      regexp: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock" #正则匹配
      replace: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock  --iptables=true" #替换后的字符串

  - name: setup docker registry
    copy:	#copy模块
      content: |	#content直接写文本内容，直接生成一个文件
        {
          "registry-mirrors":["https://reg.foldwin.org"] #镜像加速地址
        }
      dest: /etc/docker/daemon.json	#生成的文件路径

  - name: setup docker certs
    file:	#file文件管理模块
      path: /etc/docker/certs.d/reg.foldwin.org #远端路径
      state: directory	#如果这个目录不存在就创建目录


  - name: cp certs file	
    copy:
      src: "{{item.src}}"
      dest: "{{item.dest}}"
    loop:	#ansible循环，依次读取列表里面每一条字典，循环变量item
      - src: /etc/docker/certs.d/reg.foldwin.org/ca.crt
        dest: /etc/docker/certs.d/reg.foldwin.org/ca.crt

      - src: /etc/docker/certs.d/reg.foldwin.org/ca.crt
        dest: /etc/pki/ca-trust/source/anchors/ca.crt

  - name: load module
    copy:
      dest: "{{item.dest}}"
      content: "{{item.value}}"
    loop:
      - dest: /etc/modules-load.d/docker_mod.conf
        value: br_netfilter
      - dest: /etc/sysctl.d/docker.conf
        value: |
          net.bridge.bridge-nf-call-iptables = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          net.ipv4.ip_forward = 1

  - name: update ca cert
    shell: "update-ca-trust extract;sysctl --system"[devops@harbor ansible]$ vim k8s-devops.yml
- name: devops k8s from ansible
  hosts: all
  tasks:
  - name: setup docker repo
    yum_repository:
      name: docker
      description: docker
      baseurl: http://172.25.254.24:4444/docker
      file: docker
      gpgcheck: no

  - name: install docker
    dnf:
      name: docker-ce
      state: present

  - name: copy docker.service
    shell: cp /lib/systemd/system/docker.service /etc/systemd/system/docker.service
    args:
      creates: /etc/systemd/system/docker.service

  - name: setup docker.service
    replace:
      path: /etc/systemd/system/docker.service
      regexp: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock"
      replace: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock  --iptables=true"

  - name: setup docker registry
    copy:
      content: |
        {
          "registry-mirrors":["https://reg.foldwin.org"]
        }
      dest: /etc/docker/daemon.json

  - name: setup docker certs
    file:
      path: /etc/docker/certs.d/reg.foldwin.org
      state: directory


  - name: cp certs file
    copy:
      src: "{{item.src}}"
      dest: "{{item.dest}}"
    loop:
      - src: /etc/docker/certs.d/reg.foldwin.org/ca.crt
        dest: /etc/docker/certs.d/reg.foldwin.org/ca.crt

      - src: /etc/docker/certs.d/reg.foldwin.org/ca.crt
        dest: /etc/pki/ca-trust/source/anchors/ca.crt

  - name: load module
    copy:
      dest: "{{item.dest}}"
      content: "{{item.value}}"
    loop:
      - dest: /etc/modules-load.d/docker_mod.conf
        value: br_netfilter
      - dest: /etc/sysctl.d/docker.conf
        value: |
          net.bridge.bridge-nf-call-iptables = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          net.ipv4.ip_forward = 1

  - name: update ca cert
    shell: "update-ca-trust extract;sysctl --system"

  - name: start services
    service:	#系统服务管理模块
      name: "{{item}}"
      state: restarted	#重启服务
      enabled: yes	#设置开机自启动
    loop:
      - docker
      - systemd-modules-load.service
[devops@harbor ansible]$ ansible-playbook k8s-devops.yml 

6.利用ansible安装cri-dockerd

[devops@harbor ~]$ tar zxf cri-dockerd-0.4.4.amd64.tgz 
[devops@harbor ~]$ ls
ansible  cri-dockerd  cri-dockerd-0.4.4.amd64.tgz

[devops@harbor ~]$ cd cri-dockerd/
[devops@harbor cri-dockerd]$ ls
cri-dockerd  cri-docker.service  cri-docker.socket

[devops@harbor cri-dockerd]$ vim cri-docker.service 
[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --pod-infra-container-image=reg.foldwin.org/k8s/pause:3.10.1 
#--network-plugin=cni 开启cni网络插件，k8s网络必须
#--pod-infra-container-image=reg.foldwin.org/k8s/pause:3.10.1 指定pause镜像
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

[devops@harbor ansible]$ vim k8s-devops.yml 
  - name: stup cri-dockerd
    copy:
      src: "{{item.src}}"
      dest: "{{item.dest}}"
      mode: '0755'
    loop:
      - src: ../cri-dockerd/cri-dockerd
        dest: /usr/bin/cri-dockerd

      - src: "../cri-dockerd/cri-docker.service"
        dest: "/etc/systemd/system/cri-docker.service"

      - src: "../cri-dockerd/cri-docker.socket"
        dest: "/etc/systemd/system/cri-docker.socket"

  - name: update ca cert
    shell: "update-ca-trust extract;sysctl --system;systemctl daemon-reload"

  - name: start services
    service:
      name: "{{item}}"
      state: restarted
      enabled: yes
    loop:
      - docker
      - systemd-modules-load.service
      - cri-docker
      
[devops@harbor ansible]$ ansible-playbook k8s-devops.yml 

7.利用ansible部署k8s软件

[devops@harbor ansible]$ vim k8s-devops.yml 
- name: install k8s from ansible
  hosts: all
  tasks:
  - name: setup k8s repo
    yum_repository:
      name: kubernetes
      description: kubernetes
      baseurl: https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm
      file: kubernetes
      gpgcheck: no

  - name: install software in master
    dnf:
      name: "{{item}}"
      state: present
    loop:
      - kubeadm-1.35.7-150500.1.1
      - kubelet-1.35.7-150500.1.1
      - kubectl-1.35.7-150500.1.1
    when: inventory_hostname == "172.25.254.25" #条件判断，只有master主机才执行该任务

  - name: install software in work node
    dnf:
      name: "{{item}}"
      state: present
    loop:
      - kubeadm-1.35.7-150500.1.1
      - kubelet-1.35.7-150500.1.1
    when: inventory_hostname == "172.25.254.26" or inventory_hostname == "172.25.254.27"

  - name: setup kubectl kubeadm complication
    lineinfile:
      path: /root/.bashrc
      line: | #|是保留换行，没有|则会变成一整行
        source <(kubectl completion bash)	#bash命令补齐语句
        source <(kubeadm completion bash)
    when: inventory_hostname == "172.25.254.25"

  - name: start kubelet
    service:
      name: kubelet
      state: started
      enabled: yes

[devops@harbor ansible]$ ansible-playbook k8s-devops.yml 

#让补齐功能生效
[root@master mnt]# source  ~/.bashrc
[root@master mnt]# kubectl 命令后按2次tab键
alpha          (Commands for features in alpha)
annotate       (更新一个资源的注解)
api-resources  (Print the supported API resources on the server)
api-versions   (Print the supported API versions on the server, in the form of "group/version")
apply          (Apply a configuration to a resource by file name or stdin)
attach         (挂接到一个运行中的容器)
auth           (Inspect authorization)
autoscale      (Auto-scale a deployment, replica set, stateful set, or replication controller)
certificate    (Modify certificate resources)

[root@master mnt]# kubeadm
certs       config      init        kubeconfig  token       version
completion  help        join        reset       upgrade


[root@master mnt]# systemctl status kubelet.service
● kubelet.service - kubelet: The Kubernetes Node Agent
     Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; preset: disabled)
    Drop-In: /usr/lib/systemd/system/kubelet.service.d
             └─10-kubeadm.conf
     Active: activating (auto-restart) (Result: exit-code) since Tue 2026-08-18 17:09:25 CST; 1s ago
       Docs: https://kubernetes.io/docs/
    Process: 79433 ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_>
   Main PID: 79433 (code=exited, status=1/FAILURE)

四、下载k8s集群所需的镜像

1.查看镜像

[root@master ~]# kubeadm config images list 
I0829 16:04:27.220217   49948 version.go:260] remote version is much newer: v1.37.0; falling back to: stable-1.35
W0829 16:04:37.224928   49948 version.go:108] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.35.txt": Get "https://dl.k8s.io/release/stable-1.35.txt": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
W0829 16:04:37.224957   49948 version.go:109] falling back to the local client version: v1.35.7
registry.k8s.io/kube-apiserver:v1.35.7
registry.k8s.io/kube-controller-manager:v1.35.7
registry.k8s.io/kube-scheduler:v1.35.7
registry.k8s.io/kube-proxy:v1.35.7
registry.k8s.io/coredns/coredns:v1.13.1
registry.k8s.io/pause:3.10.1
registry.k8s.io/etcd:3.6.6-0

2.下载镜像

[root@master ~]#  kubeadm config images pull \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.35.7 \
--cri-socket=unix:///var/run/cri-dockerd.sock
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.35.7
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.35.7
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.35.7
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.35.7
[config/images] Pulled registry.aliyuncs.com/google_containers/coredns:v1.13.1
[config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.10.1
[config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.6.6-0

3.上传镜像到本地harbor

[root@master ~]# docker login reg.foldwin.org -u admin
Password: 

WARNING! Your credentials are stored unencrypted in '/root/.docker/config.json'.
Configure a credential helper to remove this warning. See
https://docs.docker.com/go/credential-store/

Login Succeeded

[root@master ~]# docker images  --format  "{{.Repository}}:{{.Tag}}" | awk -F "/" '/google/{system("docker tag "$0" reg.foldwin.org/k8s/"$3)}'
#--format 自定义输出格式，只打印仓库名：标签
#-F 指定分隔符为斜杠 /
#system调用shell执行命令

[root@master ~]# docker images  --format  "{{.Repository}}:{{.Tag}}" | awk -F "/" '/foldwin/{system("docker push "$0)}'
#$0 代表全名

五、在master中初始化K8s集群

1. 在master中完成集群初始化

[root@master ~]#  kubeadm init --pod-network-cidr=10.244.0.0/16 \
#--pod-network-cidr Pod网络网段，必须要与flannel yaml pod网段保持一致
#10.244.0.0/16 是网络插件的默认pod网段
--image-repository reg.foldwin.org/k8s \
#镜像仓库前缀地址，里面得有k8s所需镜像
--kubernetes-version v1.35.7 \
#指定k8s集群版本
--cri-socket=unix:///var/run/cri-dockerd.sock
#指定套接字

#其他主机加入本集群的凭证，每个人不一样
kubeadm join 172.25.254.25:6443 --token mnaxlf.8hafk3wi5lbqgc1e \
	--discovery-token-ca-cert-hash sha256:3f7864d9ddfa136f81266bae965b944a886472876e668fcf31b261ea4c9212a4
    
#如果忘记了
[root@master ~]# kubeadm token create --print-join-command 
kubeadm join 172.25.254.25:6443 --token vr1xac.2yo3pyppukgbfssk --discovery-token-ca-cert-hash sha256:3f7864d9ddfa136f81266bae965b944a886472876e668fcf31b261ea4c9212a4 

#如果初始化出问题，需要重置集群设定
[root@master ~]# kubeadm reset  --cri-socket=unix:///var/run/cri-dockerd.sock

2. 添加k8s环境变量到本机

[root@master ~]# echo "export KUBECONFIG=/etc/kubernetes/admin.conf" > ~/.bash_profile
#KUBECONFIG 指定集群管理员配置文件路径
#admin.conf 集群最高权限配置文件
[root@master ~]# source ~/.bash_profile 
[root@master ~]# kubectl get nodes
NAME     STATUS     ROLES           AGE     VERSION
master   NotReady   control-plane   3m40s   v1.35.7

3. 添加node节点到本集群

[root@node2 ~]# kubeadm join 172.25.254.25:6443 --token vr1xac.2yo3pyppukgbfssk --discovery-token-ca-cert-hash sha256:3f7864d9ddfa136f81266bae965b944a886472876e668fcf31b261ea4c9212a4 --cri-socket=unix:///var/run/cri-dockerd.sock

#测试
[root@master ~]# kubectl get nodes 
NAME     STATUS     ROLES           AGE    VERSION
master   NotReady   control-plane   7m4s   v1.35.7
node1    NotReady   <none>          37s    v1.35.7
node2    NotReady   <none>          27s    v1.35.7

4.安装网络插件

[root@master ~]# tar zxf kube-flannel-v0.28.9.tar.gz 
[root@master ~]# ls
anaconda-ks.cfg  flannel  kube-flannel-v0.28.9.tar.gz

[root@master ~]# cd flannel/
[root@master flannel]# ls
kube-flannel-v0.28.9.tar  kube-flannel.yml

[root@master flannel]# docker load -i kube-flannel-v0.28.9.tar
Loaded image: ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-flannel3
Loaded image: ghcr.io/flannel-io/flannel:v0.28.9

[root@master flannel]# docker tag ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-flannel3 reg.foldwin.org/flannel-io/flannel-cni-plugin:v1.9.1-flannel3

[root@master flannel]# docker tag ghcr.io/flannel-io/flannel:v0.28.9 reg.foldwin.org/flannel-io/flannel:v0.28.9 

[root@master flannel]# docker push reg.foldwin.org/flannel-io/flannel-cni-plugin:v1.9.1-flannel3

[root@master flannel]# docker push reg.foldwin.org/flannel-io/flannel:v0.28.9

[root@master ~]# cd flannel/
[root@master flannel]# grep -n image: kube-flannel.yml
148:        image: reg.foldwin.org/flannel-io/flannel:v0.28.9
175:        image: reg.foldwin.org/flannel-io/flannel-cni-plugin:v1.9.1-flannel3
186:        image: reg.foldwin.org/flannel-io/flannel:v0.28.9
[root@master flannel]# ll
总用量 33960
-rw------- 1 root root 34765312  8月 20 10:58 kube-flannel-v0.28.9.tar
-rw-r--r-- 1 root root     4476  8月  7 18:09 kube-flannel.yml

[root@master flannel]# kubectl apply -f kube-flannel.yml 
namespace/kube-flannel created
serviceaccount/flannel created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created

[root@master flannel]# kubectl get nodes
NAME     STATUS     ROLES           AGE     VERSION
master   Ready      control-plane   16m     v1.35.7
node1    Ready   <none>          9m56s   v1.35.7
node2    Ready   <none>          9m46s   v1.35.7

六、制作业务镜像：自定义Nginx Web应用

1.制作镜像

[root@harbor ~]# mkdir -p /opt/nginx-web-build
[root@harbor ~]# cd /opt/nginx-web-build/
[root@harbor nginx-web-build]# mkdir base-io
[root@harbor nginx-web-build]# cd base-io/
[root@harbor base-io]# docker load -i nginx-1.26.tar
Loaded image: nginx:1.26

[root@harbor nginx-web-build]# vim Dockerfile #构建镜像的指令文件
FROM nginx:1.26 AS builder
RUN echo "<h1>SRE K8s Project Demo | 高可用web业务集群项目</h1>" > /usr/share/nginx/html/index.html
#FROM 指定基础镜像 AS 构建阶段名称
#RUN:在镜像构建容器内执行shell命令

FROM nginx:1.26	
COPY --from=builder /usr/share/nginx/html/index.html /usr/share/nginx/html/index.html
#开启第二阶段，builder阶段会被丢弃，不会进入最终镜像，减小镜像体积
#COPY --from=builder 从builder构建阶段里的容器里面复制文件，就不需要再安装echo等工具
EXPOSE 80	#声明暴露的端口，但不会自动实际打开端口
CMD ["nginx","-g","daemon off;"] #容器启动默认执行命令 nginx以前台方式运行
#docker容器必须前台进程，进程退出容器就结束了，nginx是默认后台守护进程运行，容器就会立刻退出，所以用daemon off 让nginx留在前台，保证容器正常运行

[root@harbor nginx-web-build]# docker build -t reg.foldwin.org/nginx-web-project/nginx-web:v1.0 .

2.测试本地镜像，并上传到harbor仓库

[root@harbor nginx-web-build]# docker run -d --name test-web -p 8080:80 reg.foldwin.org/nginx-web-project/nginx-web:v1.0 
ea74892534d14d94e4cf226e9ebc672568881233fff01aebc81ee08faf11b151
#-d 虽然再后台运行，但是nginx不会执行fork退出父程序因为配置了daemon off ，而是让父程序一直在运行，然后依然生成多个worker子程序处理请求
[root@harbor nginx-web-build]# curl 127.0.0.1:8080
<h1>SRE K8s Project Demo | 高可用web业务集群项目</h1>
[root@harbor nginx-web-build]# docker rm -f test-web 
test-web

[root@harbor nginx-web-build]# docker push reg.foldwin.org/nginx-web-project/nginx-web:v1.0

七、k8s资源yaml清单

1.ConfigMap：Nginx配置

[root@master nginx-web-project]# vim configmap-nginx.yaml 
apiVersion: v1
kind: ConfigMap	
metadata:
  name: nginx-conf	
  namespace: nginx-web-project
data:
  nginx.conf: |
    user nginx;
    worker_processes auto;
    events { worker_connections 1024; }
    http {
      server {
        listen 80;
        root /usr/share/nginx/html;
        index index.html;
      }
    }

2.Deployment：业务控制器

重点：存活探针 livenessProbe、就绪探针 readinessProbe、资源 request/limit、滚动更新策略

[root@master nginx-web-project]# vim deployment-web.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-web
  namespace: nginx-web-project
spec:
  replicas: 3	#期望副本
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1	#最多可以比期望副本多启动一个临时Pod
      maxUnavailable: 0	#更新期间，不允许删除任何旧Pod，旧Pod必须等新Pod就绪之后才销毁
#效果：更新业务时服务零宕机，不会有业务Pod下线
  selector:
    matchLabels:
      app: nginx-web
  template:
    metadata:
      labels:
        app: nginx-web
    spec:
      containers:
      - name: nginx-web
        image: reg.foldwin.org/nginx-web-project/nginx-web:v1.0
        ports:
        - containerPort: 80
        resources:
          requests:	#调度请求资源
            cpu: 100m
            memory: 128Mi
          limits:	#资源请求上线
            cpu: 200m
            memory: 256Mi
        livenessProbe: #存活探针：判断Pod是否活着
          httpGet:
            path: /
            port: 80
          initialDelaySeconds: 5	#Pod启动5秒后开始探测
          periodSeconds: 10	#每10秒探测一次
#如果连续失败:k8s认为Pod卡死异常，直接重启Pod做自愈
        readinessProbe:	#就绪探针：判断Pod业务是否就绪，可以接受流量
          httpGet:
            path: /
            port: 80
          initialDelaySeconds: 3 #启动3秒后开始探测
          periodSeconds: 5	#每5秒探测一次
#如果没准备好就不会给Pod业务流量
        volumeMounts:
        - name: conf-volume
          mountPath: /etc/nginx/nginx.conf
          subPath: nginx.conf #只挂载nginx.conf这一个文件，如果不加subPath会把/etc/nginx整个目录替换，导致原有其他配置文件全部丢失
      volumes:
      - name: conf-volume
        configMap:
          name: nginx-conf

3.Service服务暴露

[root@master nginx-web-project]# vim service-web.yaml
apiVersion: v1
kind: Service
metadata:
  name: nginx-web-svc
  namespace: nginx-web-project
spec:
  selector:
    app: nginx-web
  ports:
  - port: 80 #service自身端口
    targetPort: 80	#pod容器内部端口
  type: ClusterIP	

4.Ingress Nginx，对外提供HTTPS

构建metallb

• 上传镜像

[root@master services]# tar zxf metallb-v0.16.1.tar.gz
[root@master metaLLB]# docker load  -i metallb-v0.16.1.tar
[root@master metaLLB]# docker tag quay.io/metallb/controller:v0.16.1 reg.foldwin.org/metallb/controller:v0.16.1
[root@master metaLLB]# docker tag quay.io/metallb/speaker:v0.16.1 reg.foldwin.org/metallb/speaker:v0.16.1
[root@master metaLLB]# docker push  reg.foldwin.org/metallb/controller:v0.16.1
[root@master metaLLB]# docker push reg.foldwin.org/metallb/speaker:v0.16.1

• 更改kube-proxy

[root@master mateLLB]# kubectl edit configmap -n kube-system kube-proxy
configmap/kube-proxy edited
59     mode: "ipvs"	#工作模式切换为ipvs，默认是iptables模式，ipvs内核模块，性能更高，支持更多负载均衡的算法，大规模集群推荐ipvs
60     ipvs:
61       strictARP: true	#开启严格ARP，MetalLB必须依赖这个参数


[root@master metaLLB]# kubectl -n kube-system get pods
NAME                             READY   STATUS    RESTARTS   AGE
coredns-d77667cf9-5nqqv          1/1     Running   0          177m
coredns-d77667cf9-sbp4t          1/1     Running   0          177m
etcd-master                      1/1     Running   0          177m
kube-apiserver-master            1/1     Running   0          177m
kube-controller-manager-master   1/1     Running   0          177m
kube-proxy-2zblf                 1/1     Running   0          170m
kube-proxy-czl6w                 1/1     Running   0          170m
kube-proxy-gm8xz                 1/1     Running   0          177m
kube-scheduler-master            1/1     Running   0          177m

[root@master metaLLB]# kubectl -n kube-system delete pods kube-proxy-2zblf kube-proxy-czl6w kube-proxy-gm8xz 	#删除旧pod，新pod重新读取配置
pod "kube-proxy-2zblf" deleted from kube-system namespace
pod "kube-proxy-czl6w" deleted from kube-system namespace
pod "kube-proxy-gm8xz" deleted from kube-system namespace

[root@master metaLLB]# kubectl -n kube-system get pods
NAME                             READY   STATUS    RESTARTS   AGE
coredns-d77667cf9-5nqqv          1/1     Running   0          178m
coredns-d77667cf9-sbp4t          1/1     Running   0          178m
etcd-master                      1/1     Running   0          178m
kube-apiserver-master            1/1     Running   0          178m
kube-controller-manager-master   1/1     Running   0          178m
kube-proxy-4b2m6                 1/1     Running   0          10s
kube-proxy-97nwj                 1/1     Running   0          10s
kube-proxy-j4dcq                 1/1     Running   0          10s
kube-scheduler-master            1/1     Running   0          178m

• 更改metallb的配置文件

[root@master metaLLB]# vim 1-metallb-native.yaml
2136:        image: metallb/controller:v0.16.1
2233:        image: metallb/speaker:v0.16.1


[root@master metaLLB]# vim 2-metallb-confmap.yml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: first-pool
  namespace: metallb-system
spec:
  addresses:
  - 172.25.254.50-172.25.254.99


---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: example
  namespace: metallb-system
spec:
  ipAddressPools:
  - first-pool
  
[root@master metaLLB]# kubectl apply -f 2-metallb-confmap.yml  -f 1-metallb-native.yaml

部署ingress-nginx控制器

• 下载deploy文件

wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.15.1/deploy/static/provider/baremetal/deploy.yaml

• 准备ingress所需镜像

[root@master ingress-v1.5.1]# docker load -i ingress-v1.15.1.tar
Loaded image: registry.k8s.io/ingress-nginx/controller:v1.15.1
Loaded image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.6.9
[root@master ingress-v1.5.1]# docker tag registry.k8s.io/ingress-nginx/controller:v1.15.1 reg.foldwin.org/ingress-nginx/controller:v1.15.1

[root@master ingress-v1.5.1]# docker tag registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.6.9 reg.foldwin.org/ingress-nginx/kube-webhook-certgen:v1.6.9

[root@master ingress-v1.5.1]# docker push reg.foldwin.org/ingress-nginx/controller:v1.15.1

[root@master ingress-v1.5.1]# docker push reg.foldwin.org/ingress-nginx/kube-webhook-certgen:v1.6.9

• 更改deploy.yaml

[root@master ingress-v1.5.1]# vim deploy.yaml 
444:        image: reg.foldwin.org/ingress-nginx/controller:v1.15.1
547:        image: reg.foldwin.org/ingress-nginx/kube-webhook-certgen:v1.6.9
603:        image: reg.foldwin.org/ingress-nginx/kube-webhook-certgen:v1.6.9
365   		type: LoadBalancer

[root@master ingress-v1.5.1]# kubectl apply -f deploy.yaml 
[root@master metaLLB]# kubectl -n ingress-nginx get svc
NAME                                 TYPE           CLUSTER-IP       EXTERNAL-IP     PORT(S)                      AGE
ingress-nginx-controller             LoadBalancer   10.110.14.255    172.25.254.50   80:30628/TCP,443:32363/TCP   23m
ingress-nginx-controller-admission   LoadBalancer   10.103.175.200   172.25.254.51   443:30973/TCP                23m

编写ingress-web

[root@master nginx-web-project]# vim ingress-web.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-web-ingress
  namespace: nginx-web-project
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec:
  tls:
  - hosts:
    - web.foldiwn.org
    secretName: nginx-web-tls-secret
  rules:
  - host: web.foldwin.org
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nginx-web-svc
            port:
              number: 80

八、实际操作

1.版本更新

#构建新版本镜像
[root@harbor nginx-web-build]# docker build -t reg.foldwin.org/nginx-web-project/nginx-web:v2.0 .

[root@harbor nginx-web-build]# docker push reg.foldwin.org/nginx-web-project/nginx-web:v2.0

[root@master ~]# kubectl set image deployment/nginx-web nginx-web=reg.foldwin.org/nginx-web-project/nginx-web:v2.0 -n nginx-web-project 
deployment.apps/nginx-web image updated

#观察滚动更新状态
[root@master ~]# kubectl rollout status deployment nginx-web -n nginx-web-project
Waiting for deployment "nginx-web" rollout to finish: 2 out of 3 new replicas have been updated...
Waiting for deployment "nginx-web" rollout to finish: 2 out of 3 new replicas have been updated...
Waiting for deployment "nginx-web" rollout to finish: 2 out of 3 new replicas have been updated...
Waiting for deployment "nginx-web" rollout to finish: 1 old replicas are pending termination...
Waiting for deployment "nginx-web" rollout to finish: 1 old replicas are pending termination...
Waiting for deployment "nginx-web" rollout to finish: 1 old replicas are pending termination...
deployment "nginx-web" successfully rolled out

#查看更新历史
[root@master ~]# kubectl rollout history deployment nginx-web -n nginx-web-project 
deployment.apps/nginx-web 
REVISION  CHANGE-CAUSE
1         <none>
2         <none>

#故障回滚至上一个版本
[root@master ~]# kubectl rollout undo deployment nginx-web -n nginx-web-project
deployment.apps/nginx-web rolled back