K8s1.28 全栈落地｜Jenkins+GitLab+Harbor DevOps 流水线封神实战

kubernetes与devops整合

在jenkins服务器的/usr/local/docker/jenkins_docker/data/workspace/mytest下

deploy.sh脚本优化如下：

#! /bin/bash
harbor_ip=$1
harbor_repo=$2
project=$3
version=$4
container_port=$5
host_port=$6
imageName=$harbor_ip/$harbor_repo/$project:$version
echo $imageName
#如果当前容器正在运行，则删除
container_id=`docker ps -a | grep ${project} | awk '{print $1}'`
echo $container_id
if [ "$container_id" != "" ]
then
	docker stop $container_id
	docker rm $container_id
fi
#如果当前镜像已存在，则删除
tag=`docker images | grep ${project} | awk '{print $2}'`
echo $tag
if [[ "$tag" =~ "$version" ]]
then
	docker rmi $imageName
fi
#非本地登录镜像仓库**********此处优化
if grep -q -- "--insecure-registry" /lib/systemd/system/docker.service; then
	echo "已配置过"
else
	echo "初次配置"
	cp /lib/systemd/system/docker.service /lib/systemd/system/docker.service.bak
	sed -i 's|ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock|ExecStart=/usr/bin/dockerd -H fd:// --insecure-registry 10.1.8.11 --containerd=/run/containerd/containerd.sock|g' /lib/systemd/system/docker.service
	systemctl daemon-reload && systemctl restart docker
fi
docker login -u admin -p harbor123 $harbor_ip
#********************优化至此处
#拉取镜像
docker pull $imageName
#运行容器
docker run -d -p $host_port:$container_port --name $project $imageName
echo "SUCCESS"

先运行mytest，再运行pipeline，完美闭环

kubernetes部署

1：前置环境

主机名解析

[root@master ~ 15:22:29]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.1.8.14   master
10.1.8.15   node1
[root@node1 ~ 15:24:05]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.1.8.14   master
10.1.8.15   node1

防火墙关闭，禁用selinux

[root@master ~]# systemctl disable firewalld.service --now
[root@master ~]# setenforce 0
[root@node1 ~]# systemctl disable firewalld.service --now
[root@node1 ~]# setenforce 0

配置固定IP

可以直接nmtui或者nmcli设置静态地址

master

[root@master ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens32
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens32
UUID=94dc7f1d-dbe5-4209-9d70-b9605e7f146d
DEVICE=ens32
ONBOOT=yes
IPADDR=10.1.8.14
PREFIX=24
GATEWAY=10.1.8.2
DNS1=223.5.5.5
DNS2=223.6.6.6
IPV6_PRIVACY=no

node1

TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens32
UUID=94dc7f1d-dbe5-4209-9d70-b9605e7f146d
DEVICE=ens32
ONBOOT=yes
IPADDR=10.1.8.15
PREFIX=24
GATEWAY=10.1.8.2
DNS1=223.5.5.5
DNS2=223.6.6.6
IPV6_PRIVACY=no

2：安装环境包

#所有节点运行
[root@master ~]#  yum -y install  conntrack ntpdate ntp ipvsadm ipset iptables curl sysstat libseccomp unzip wget net-tools tree git psmisc telnet unzip gcc gcc-c++ make

3： 关闭NetworkManager（可选）

#所有节点运行
[root@master ~ 13:03:45]#  systemctl stop NetworkManager
[root@master ~ 13:03:55]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
Removed symlink /etc/systemd/system/network-online.target.wants/NetworkManager-wait-online.service.

4：调整系统内核参数（注意：升级内核）

#所有节点执行
[root@master ~]# yum update -y kernel && reboot
[root@master ~]# cat >/etc/sysctl.d/kubernetes.conf<<EOF
# 开启Linux内核的网络桥接功能，同时启用iptables和ip6tables的网络包过滤功能，用于在网络桥接时进行网络包过滤
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
# 开启路由转发，转发IPv4的数据包
net.ipv4.ip_forward=1
# 尽可能避免使用交换分区，提升k8s性能
vm.swappiness=0
# 不检查物理内存是否够用
vm.overcommit_memory=1
EOF

立即生效

[root@master ~]# sysctl --system

5：关闭 swap

#所有节点执行
[root@master ~] swapoff -a
[root@master ~]vim /etc/fstab
#注释掉swap 那一行

6：时间同步

[root@master ~]# yum -y install chrony
[root@master ~]# systemctl restart chronyd\
#列出 Chrony 守护进程当前配置和使用的所有时间源（NTP 服务器）及其同步状态信息
[root@master ~]# chronyc sources -v
#将硬件时钟的时间同步到系统时钟
[root@master ~]# hwclock -s                    

7：IPVS功能

这个文件 /etc/modules‑load.d/ipvs.conf 用来开机自动加载一批 Linux 内核模块，给 Kubernetes Service 的 ipvs 负载均衡模式做准备

[root@master ~]# cat >>/etc/modules-load.d/ipvs.conf<<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
overlay
br_netfilter
EOF
#重启服务
[root@master ~]# systemctl restart systemd-modules-load
#查看内核模块
[root@master ~]# lsmod | grep -e ip_vs -e nf_conntrack_ipv4

8：docker-ce环境

#master节点
[root@master ~]# yum install -y yum-utils
#使用阿里云镜像
[root@master ~]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#安装docker-ce
#❗❗❗这里可以直接安装containerd.io，因为k8s1.28版本使用containerd作为容器运行时，参考下面的node节点的部署
[root@master ~]# yum install -y docker-ce
#配置镜像加速
[root@master ~ 13:22:57]# cat /etc/docker/daemon.json 
{
  "registry-mirrors": 	 ["https://e8e019c4e9b142a78961a95c6da13a9f.mirror.swr.myhuaweicloud.com"],
  "insecure-registries":["10.1.8.11:80"]
}

#node节点
[root@node1 ~]yum install -y yum-utils
[root@node1 ~]yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
[root@node1 ~]yum install -y containerd.io

9：配置k8s的yum源

#所有节点执行
[root@master ~] cat <<EOF>/etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg,http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
[root@master ~]# yum clean all
[root@master ~]# yum makecache

10：k8s软件安装及配置

#所有节点执行
#安装1.28.2-0版本
[root@master ~]# yum install -y kubeadm-1.28.0-0 kubelet-1.28.0-0 kubectl-1.28.0-0

[root@master ~ 13:38:04]# mv /etc/containerd/config.toml /etc/containerd/config.toml.bak
[root@master ~ 13:40:04]# containerd config default > /etc/containerd/config.toml
#配置镜像加速
[root@master ~ 13:40:09]# vim /etc/containerd/config.toml
#追加你的harbor私有仓库配置（放在 `[plugins."io.containerd.grpc.v1.cri".registry]` 大括号下面）
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."10.1.8.11:80"]
  endpoint = ["http://10.1.8.11:80"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."10.1.8.11:80".tls]
  insecure_skip_verify = true
#修改
#Kubernetes 每一个 Pod 启动的时候，一定会先启动一个 pause 容器（沙箱容器）负责占着网络、IP
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"
#指定containerd使用systemd cgroup驱动来管理容器资源
SystemdCgroup = true

[root@node1 ~ 13:50:19]# mkdir -p /etc/containerd
[root@node1 ~ 13:51:09]# containerd config default > /etc/containerd/config.toml
[root@node1 ~ 13:51:15]# vim /etc/containerd/config.toml
#追加你的 harbor 私有仓库配置（放在 `[plugins."io.containerd.grpc.v1.cri".registry]` 大括号下面）
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."10.1.8.11:80"]
  endpoint = ["http://10.1.8.11:80"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."10.1.8.11:80".tls]
  insecure_skip_verify = true
#修改
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"
SystemdCgroup = true

#所有节点执行
[root@master ~ 13:55:34]# systemctl daemon-reload
[root@master ~ 13:58:06]# systemctl restart containerd
[root@master ~ 13:58:11]# systemctl enable containerd

永久设置 crictl 的运行时端点

#所有节点执行
[root@master ~]# cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
[root@master ~]# crictl info
#正常输出一大段 JSON 代表 containerd‑CRI 已经正常工作

11：集群初始化

⚠注意：只在master节点做操作

[root@master ~ 13:58:18]# kubeadm config images pull \
> --kubernetes-version=v1.28.0 \
> --image-repository=registry.aliyuncs.com/google_containers

[config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.9
[config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.5.9-0
[config/images] Pulled registry.aliyuncs.com/google_containers/coredns:v1.10.1
[root@master ~ 14:05:56]# crictl images
IMAGE                                                             TAG                 IMAGE ID            SIZE
registry.aliyuncs.com/google_containers/coredns                   v1.10.1             ead0a4a53df89       16.2MB
registry.aliyuncs.com/google_containers/etcd                      3.5.9-0             73deb9a3f7025       103MB
registry.aliyuncs.com/google_containers/kube-apiserver            v1.28.0             bb5e0dde9054c       34.6MB
registry.aliyuncs.com/google_containers/kube-controller-manager   v1.28.0             4be79c38a4bab       33.4MB
registry.aliyuncs.com/google_containers/kube-proxy                v1.28.0             ea1030da44aa1       24.6MB
registry.aliyuncs.com/google_containers/kube-scheduler            v1.28.0             f6f496300a2ae       18.8MB
registry.aliyuncs.com/google_containers/pause                     3.9                 e6f1816883972       322kB

#创建初始化集群配置文件
[root@master ~]# kubeadm config print init-defaults > kubeadm-init.yaml
#修改初始化配置文件
[root@master ~]# vim kubeadm-init.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.1.8.14 #主机地址
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: master   #修改master节点主机名
  taints:        # 去掉 Null
  - effect: NoSchedule #添加污点
    key: node-role.kubernetes.io/control-plane #标记这台机器是控制平面 (master) 节点
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers #切换成国内镜像仓库地址
kind: ClusterConfiguration
kubernetesVersion: 1.28.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.100.0.0/16  #增加pod网段
scheduler: {}

#初始化
[root@master ~]# kubeadm init --config=kubeadm-init.yaml --upload-certs | tee kubeadm-init.log
....
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.1.8.14:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:76b223aad44ccd36aeb6866a1428857445dddc0c4c2d245004cbbf010b96eee5

[root@master ~]# mkdir -p $HOME/.kube
[root@master ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master ~]# chown $(id -u):$(id -g) $HOME/.kube/config
#当前未配置网络CNI组件
[root@master ~ 14:33:33]# kubectl get nodes
NAME     STATUS     ROLES           AGE    VERSION
master   NotReady   control-plane   4m1s   v1.28.0
[root@master ~ 14:33:38]# kubectl get pods -n kube-system
NAME                             READY   STATUS    RESTARTS   AGE
coredns-66f779496c-5x5q4         0/1     Pending   0          3m55s
coredns-66f779496c-gncfp         0/1     Pending   0          3m55s
etcd-master                      1/1     Running   0          4m10s
kube-apiserver-master            1/1     Running   0          4m10s
kube-controller-manager-master   1/1     Running   0          4m10s
kube-proxy-xl9t2                 1/1     Running   0          3m55s
kube-scheduler-master            1/1     Running   0          4m10s

12：node节点加入集群

[root@node1 ~]# kubeadm join 10.1.8.14:6443 --token abcdef.0123456789abcdef \
>         --discovery-token-ca-cert-hash sha256:76b223aad44ccd36aeb6866a1428857445dddc0c4c2d245004cbbf010b96eee5
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

[root@master ~ 14:35:32]# kubectl get nodes
NAME     STATUS     ROLES           AGE     VERSION
master   NotReady   control-plane   9m44s   v1.28.0
node1    NotReady   <none>          99s     v1.28.0

❗❗❗当集群初始化失败或者node加入失败，可能是swap没有关闭

13：网络CNI组建部署

#下载Calico文件
[root@master ~ 15:03:44]# wget https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/calico.yaml
#修改Calico文件
[root@master ~]# vim calico.yaml
# 找到4601行，去掉注释并修改
- name: CALICO_IPV4POOL_CIDR
  value: "10.100.0.0/16" #与初始化文件里面pod网段一样
#修改镜像地址
[root@master ~]# sed -i 's/docker.io/quay.io/g' calico.yaml
[root@master ~]# grep image: calico.yaml | sort | uniq
          image: quay.io/calico/cni:v3.26.1
          image: quay.io/calico/kube-controllers:v3.26.1
          image: quay.io/calico/node:v3.26.1
#部署Calico
[root@master ~ 14:43:09]#  kubectl apply -f calico.yaml
#查看Pod状态
[root@master ~ 15:13:28]# kubectl get pods -A
NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-c766c8b99-8dsxk   1/1     Running   0          77s
kube-system   calico-node-6n2gf                         1/1     Running   0          77s
kube-system   calico-node-sd5jt                         1/1     Running   0          77s
kube-system   coredns-66f779496c-5x5q4                  1/1     Running   0          43m
kube-system   coredns-66f779496c-gncfp                  1/1     Running   0          43m
kube-system   etcd-master                               1/1     Running   0          43m
kube-system   kube-apiserver-master                     1/1     Running   0          43m
kube-system   kube-controller-manager-master            1/1     Running   0          43m
kube-system   kube-proxy-lb7np                          1/1     Running   1          35m
kube-system   kube-proxy-xl9t2                          1/1     Running   0          43m
kube-system   kube-scheduler-master                     1/1     Running   0          43m
#查看节点状态，发现全都ready
[root@master ~ 15:13:35]# kubectl get nodes
NAME     STATUS   ROLES           AGE   VERSION
master   Ready    control-plane   44m   v1.28.0
node1    Ready    <none>          36m   v1.28.0

14： 配置命令补全

# 所有节点执行
# 配置 crictl 命令自动补全
[root@master ~ 15:16:02]# crictl completion bash > /etc/bash_completion.d/crictl
[root@master ~ 15:16:11]# source /etc/bash_completion.d/crictl
# 配置 kubectl 命令补全
[root@master ~ 15:16:16]# kubectl completion bash > /etc/bash_completion.d/kubectl
[root@master ~ 15:16:23]# source /etc/bash_completion.d/kubectl
# 配置 kubeadm 命令补全
[root@master ~ 15:16:28]# kubeadm completion bash > /etc/bash_completion.d/kubeadm
[root@master ~ 15:16:32]# source /etc/bash_completion.d/kubeadm

❗如果想在实验环境下让node也有权限，可以把 Master 的 admin.conf 复制到 Node

#在master上
[root@master ~]# scp /etc/kubernetes/admin.conf root@node1:/root/admin.conf
#在node上
[root@node1 ~]# mkdir -p ~/.kube
[root@node1 ~]# cp /root/admin.conf ~/.kube/config
#检查
[root@node1 ~ 15:28:12]# kubectl get pods -A
NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-c766c8b99-8dsxk   1/1     Running   0          16m
kube-system   calico-node-6n2gf                         1/1     Running   0          16m
kube-system   calico-node-sd5jt                         1/1     Running   0          16m
kube-system   coredns-66f779496c-5x5q4                  1/1     Running   0          58m
kube-system   coredns-66f779496c-gncfp                  1/1     Running   0          58m
kube-system   etcd-master                               1/1     Running   0          59m
kube-system   kube-apiserver-master                     1/1     Running   0          59m
kube-system   kube-controller-manager-master            1/1     Running   0          59m
kube-system   kube-proxy-lb7np                          1/1     Running   1          51m
kube-system   kube-proxy-xl9t2                          1/1     Running   0          58m
kube-system   kube-scheduler-master                     1/1     Running   0          59m
[root@node1 ~ 15:28:54]# kubectl get nodes
NAME     STATUS   ROLES           AGE   VERSION
master   Ready    control-plane   59m   v1.28.0
node1    Ready    <none>          51m   v1.28.0

整合YAML文件准备

因为yaml资源文件可以在k8s上创建应用pod，所以需要提前创建yaml，拉取到k8s服务器上

在gitlab上新建YAML文件

推荐：

注意：在master优先创建secret资源，用来访问harbor凭证，否则镜像拉取会失败

kubectl create secret docker-registry harbor-secret \
--docker-server=10.1.8.11:80 \
--docker-username=admin \
--docker-password=harbor123

输入YAML文件内容：

apiVersion: apps/v1
kind: Deployment
metadata:
  name: pipeline
  labels:
	app: pipeline
spec:
  replicas: 2
  selector:
	matchLabels:
	  app: pipeline
  template:
	metadata:
	  labels:
		app: pipeline
	spec:
	  imagePullSecrets:                   #关键凭证
	  - name: harbor-secret
	  containers:
	  - name: pipeline
		image: 10.1.8.11:80/repo/pipeline:v4.0
		imagePullPolicy: Always         #最新版本
		ports:
		- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
  labels:
	app: pipeline
  name: pipeline
spec:
  selector:
	app: pipeline
  ports:
  - port: 8081
    targetPort: 8080
  type: NodePort

harbor仓库对接

在所有k8s（master，node）节点添加harbor仓库地址配置

#所有节点执行
[root@master ~]# vim /etc/containerd/config.toml

重启containerd服务

[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl restart containerd

测试k8s（master，node）节点harbor登录

[root@master ~]# crictl pull 10.1.8.11:80/repo/pipeline:v3.0 
Image is up to date for sha256:69017324dac11aedf865390f96b88ec6c6c9feb9cbd6cac6563865da85637dda
[root@master ~]# crictl images |grep pipeline
10.1.8.11:80/repo/pipeline                                        v3.0                69017324dac11       118MB
[root@node1 ~]# crictl pull 10.1.8.11:80/repo/pipeline:v3.0
Image is up to date for sha256:69017324dac11aedf865390f96b88ec6c6c9feb9cbd6cac6563865da85637dda
[root@node1 ~]# crictl images|grep pipeline
10.1.8.11:80/repo/pipeline                           v3.0                69017324dac11       118MB

YAML推送k8s服务器设置

在master节点配置推送目标目录

[root@master ~]# mkdir /usr/local/k8s
[root@master ~]# chmod 777 /usr/local/k8s/

系统管理-系统配置

点击测试，成功后，应用并保存

修改Publish Over SSH通知目标服务器步骤的Jenkinsfile的内容

到gitlab服务器上，修改Jenkinsfile内容

保存配置，并添加新标签tag，否则识别不到新添加的yml文件

!

先构建，测试yml文件推送

构建成功，查看构建日志

发现文件没有传过去

然后检查发现pipeline根本没有拉下来，所以怀疑是标签出了错误导致拉到之前的版本

所以我修改Jenkinsfile文件，在拉取git仓库代码下面加上

stage('检查Git代码') {
    steps {
        sh '''
            echo "========== 当前HEAD =========="
            git rev-parse HEAD

            echo "========== 最近提交 =========="
            git log --oneline -5

            echo "========== 当前分支 =========="
            git branch -a

            echo "========== HEAD中是否存在pipeline.yml =========="
            git ls-tree -r HEAD --name-only | grep -E '^pipeline\\.yml$' || true

            echo "========== 直接从Git读取pipeline.yml =========="
            git show HEAD:pipeline.yml | head -20

            echo "========== 工作区是否存在pipeline.yml =========="
            ls -lah pipeline.yml || true

            echo "========== 检查Sparse Checkout =========="
            git config --get core.sparseCheckout || true
        '''
    }
}

构建后发现拉到了原始版本

真正的问题就在branches: \[name: '\*tag′]，单引号不会进行Groovy的'{tag}']]，单引号不会进行 Groovy 的 `tag′]]，单引号不会进行Groovy的'{tag}` 字符串插值，所以可以修改成branches: \[name: "refs/tags/${tag}"]

再次构建，发现成功并且文件已经传送过去

到master节点上查看，已成功传递

[root@master ~]# ls /usr/local/k8s/
pipeline.yml

Jenkins使用ssh无密码登录访问k8s的master节点

因为希望使用ssh 用户名@k8s地址 kubectl apply -f /usr/local/k8s/pipeline.yml创建资源，会进行ssh

免交互

把Jenkins中的公钥内容传递给k8s的master实现免密

[root@jenkins ~]# docker exec -it jenkins bash
root@4d08830dc80c:/# cd ~
root@4d08830dc80c:~# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): `回车`
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase): `回车`
Enter same passphrase again: `回车`
Your identification has been saved in /root/.ssh/id_rsa
Your public key has been saved in /root/.ssh/id_rsa.pub
The key fingerprint is:`回车`
SHA256:UuwkjQTJAnReTgRrbqzU5pyN5RW+VpqpO9lKwGuRSic root@4d08830dc80c
The key's randomart image is:
+---[RSA 3072]----+
|oo ++*.          |
|  + B. +         |
|   = .o.=        |
|  * . .=.        |
| E % ..oS.       |
|o X X ..*        |
| o B +o*         |
|  . .oo.         |
|     ++          |
+----[SHA256]-----+
root@4d08830dc80c:~# cd .ssh/
root@4d08830dc80c:~/.ssh# ls
id_rsa  id_rsa.pub

复制公钥内容传递给master

root@4d08830dc80c:~/.ssh# cat id_rsa.pub 
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnThRb1sOLiT/RxNfyDRjHPpAn6xppIqdKxhy5v7Ew7R0/DMz2wBcoCuPRBdaQh+zt7zppmVExtWs9+TFMHfFQCinxoNS0BvXQ6PYAAF6V9/Br4ZoppBtuWrfB7QWVAnbJhBOpECQ4Q1nASpuXEbkt6G1iX2oZpAOieZ4GqYhQTmTuJpGjCK/O1UHRAHiHmZtqrIjJri9N4T/bWVK4D2O1E3xRWW8MXQbles2emgC7uphqjCc2ybiIMZ/ZKfWBEoZ7t7a3q6sivJjIuo2vawoz6zGQzmF6M8x/Kx1qPB8l0SIPy3p3S0HGycsDFve9RQxb0ELO49Fwx8PL/cIsA347I7ukrEQubvFmbi+/8c5uAqNGCHcjI/oV6YqEVTU56NFXHu24fZe9PvxrMinS6lEbwup8gUmtyXVHR8cqStXcXieZwbv1dUupV/wWfre1YY6vhBRKlprLmc9PMzXEFpjgLRbDGeKA75ifNaXu3ISOM5YvLqO2jxVnGt1kNqhZ/XE= root@4d08830dc80c

到master节点上

[root@master ~]# mkdir .ssh
[root@master ~]# cd .ssh/
[root@master .ssh]# vim authorized_keys
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnThRb1sOLiT/RxNfyDRjHPpAn6xppIqdKxhy5v7Ew7R0/DMz2wBcoCuPRBdaQh+zt7zppmVExtWs9+TFMHfFQCinxoNS0BvXQ6PYAAF6V9/Br4ZoppBtuWrfB7QWVAnbJhBOpECQ4Q1nASpuXEbkt6G1iX2oZpAOieZ4GqYhQTmTuJpGjCK/O1UHRAHiHmZtqrIjJri9N4T/bWVK4D2O1E3xRWW8MXQbles2emgC7uphqjCc2ybiIMZ/ZKfWBEoZ7t7a3q6sivJjIuo2vawoz6zGQzmF6M8x/Kx1qPB8l0SIPy3p3S0HGycsDFve9RQxb0ELO49Fwx8PL/cIsA347I7ukrEQubvFmbi+/8c5uAqNGCHcjI/oV6YqEVTU56NFXHu24fZe9PvxrMinS6lEbwup8gUmtyXVHR8cqStXcXieZwbv1dUupV/wWfre1YY6vhBRKlprLmc9PMzXEFpjgLRbDGeKA75ifNaXu3ISOM5YvLqO2jxVnGt1kNqhZ/XE= root@4d08830dc80c

master重启sshd服务

[root@master ~]# systemctl restart sshd

在Jenkins容器中测试免密执行

root@4d08830dc80c:~/.ssh# ssh root@10.1.8.14 ls /
bin
boot
dev
etc
home
lib
lib64
media
mnt
opt
proc
root
run
sbin
srv
sys
tmp
usr
var

Jenkins中设置YAML部署

在流水线语法中

输入：（注意：执行的是绝对路径）

sh 'ssh root@10.1.8.14 kubectl apply -f /usr/local/k8s/pipeline.yml'

在gitlab中增加步骤

增加步骤内容：

stage('远程执行kubectl命令'){
		    steps{
		        sh 'ssh root@10.1.8.14 kubectl apply -f /usr/local/k8s/pipeline.yml'
		    }
        }

因为拉取的项目标签是5.0，所以pipeline.yml拉取镜像的标签也要同步变更

首页内容变更

路径：src/main/java/com/chenyan/demo/controller/TestController.java

gitlab中把原来的v5.0标签删除，重新创建，命名依然写v5.0

立即构建

在master上查看pod资源

[root@master ~]# kubectl get pod -o wide 
NAME                        READY   STATUS    RESTARTS   AGE   IP               NODE    NOMINATED NODE   READINESS GATES
pipeline-7df9676796-hwl6h   1/1     Running   0          49s   10.100.166.130   node1   <none>           <none>
pipeline-7df9676796-xxfxx   1/1     Running   0          49s   10.100.166.129   node1   <none>           <none>

[root@master ~]# kubectl get svc
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP          5h
pipeline     NodePort    10.102.61.102   <none>        8081:32671/TCP   67s

在node1上查看下载的pipeline镜像

[root@node1 ~]# crictl images|grep pipeline
10.1.8.11:80/repo/pipeline                           v5.0                a15c5b0870087       118MB
[root@node1 ~]# crictl ps -a|grep pipeline
271b75a586ef0       a15c5b0870087       2 minutes ago       Running             pipeline            0                   396d0848650a8       pipeline-7df9676796-hwl6h
fef5723b22b10       a15c5b0870087       2 minutes ago       Running             pipeline            0                   e9ebb51eb8aa1       pipeline-7df9676796-xxfxx

打开node地址查看网页

http://10.1.8.15:32671/test