Mac M1 Parallels CentOS7.9 Rancher + K8S + Gitlab + Jenkins +Harbor CICD

一、资源清单

机器名称	IP地址	角色
k8s+rancher高可用部署: https://blog.csdn.net/qq_41594280/article/details/135312148
rancher	10.211.55.200	管理K8S集群
k8svip	10.211.55.199	K8S VIP
master01	10.211.55.201	K8S集群主节点
master02	10.211.55.202	K8S集群主节点
master03	10.211.55.203	K8S集群主节点
node01	10.211.55.211	K8S集群从节点
node02	10.211.55.212	K8S集群从节点
Harbor容器私服安装部署: https://blog.csdn.net/qq_41594280/article/details/135447118
harbor	10.211.55.20	docker容器私服
Gitlab代码仓库安装部署: https://blog.csdn.net/qq_41594280/article/details/135491802
gitlab	10.211.55.33	代码仓库
Jenkins安装部署: https://blog.csdn.net/qq_41594280/article/details/135442317
jenkins	10.211.55.34	持续集成工具

二、Jenkins与GitLab集成实现自动构建(可忽略)

2.0 安装Git和Maven

# 安装Git
[root@jenkins ~]# yum install -y git
[root@jenkins ~]# git --version
git version 1.8.3.1

# 安装maven,地址: https://archive.apache.org/dist/maven
wget https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.tar.gz
mkdir /opt/maven
tar xzvf apache-maven-3.8.6-bin.tar.gz -C /opt/maven

vim ~/.bash_profile
# 内容 BEGIN
export M2_HOME=/opt/maven/apache-maven-3.8.6
export PATH=$M2_HOME/bin:$PATH
# 内容 END
source ~/.bash_profile

[root@jenkins ~]# mvn -v
Apache Maven 3.8.6 (84538c9988a25aec085021c365c560670ad80f63)
Maven home: /opt/maven/apache-maven-3.8.6
Java version: 11.0.21, vendor: Oracle Corporation, runtime: /opt/java/jdk-11.0.21
Default locale: zh_CN, platform encoding: UTF-8
OS name: "linux", version: "5.11.12-300.el7.aarch64", arch: "aarch64", family: "unix"

vim /opt/maven/apache-maven-3.8.6/conf/settings.xml

<localRepository>/opt/maven/repo</localRepository>

<mirrors>
	<!--<mirror>
	      <id>maven-default-http-blocker</id>
	      <mirrorOf>external:http:*</mirrorOf>
	      <name>Pseudo repository to mirror external repositories initially using HTTP.</name>
      <url>http://0.0.0.0/</url>
      <blocked>true</blocked>
    </mirror>-->
    <mirror>
      <id>alimaven</id>
      <name>aliyun maven</name>
      <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
      <mirrorOf>central</mirrorOf>
    </mirror>
</mirrors>

2.1 下载安装Pipeline插件

2.2 Jenkins 安装插件 GitLab Plugin、Maven Integration Plugin

2.3 服务器编写测试脚本

[root@jenkins ~]# echo "echo "hello world"" >demo.sh
[root@jenkins ~]# chmod +x demo.sh
[root@jenkins ~]# ll demo.sh
-rwxr-xr-x. 1 root root 16 1月  13 16:28 demo.sh

2.4 Jenkins创建自由风格项目并填写构建脚本

2.5 配置钩子脚本

# 服务器执行安装openssl
yum install -y openssl
# 生成ssl令牌 -> d22f1292ecf0a36007ab
openssl rand -hex 10

2.6 GitLab创建项目添加钩子脚本

保存报错

解决：Settings → Network → Outbound requests

再次添加hooks

报错

解决

还不行，去掉Jenkins系统设置里的 Enable authentication for '/project' end-point

自动构建成功

三、基于Jenkins、Harbor、GitLab自动部署SpringBoot项目到K8S

3.1 创建项目

3.2 配置Jenkins

参考2.4~2.6配置即可，最终验证钩子返回200 OK

Manage Jenkins → Tools

3.3 配置Harbor

添加用户Jenkins

添加项目

项目添加成员

3.4 Jenkins和K8S Master做免密

# jenkins, 若有可不执行
ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa

[root@jenkins ~]# cat ~/.ssh/id_rsa
-----BEGIN RSA PRIVATE KEY-----
MIIEogIBAAKCAQEAwfUH1Oa1e3eAmV8yK84w7BaKYv5d82qXhi1Jmhy0St8YkKOe
OcrjPAGtkEQZCcr8Ee/GY7mjG5YT6eK/mWyJqMlqImxjC63CjNL/zhR8x0BV7c3r
tZ4iyg/61QO8cwguN0u7wwCHT9i9xQ9hBUrz1a80MmlwjUCobLu1U0kCw32YEerm
Ux6xPkP3Wyin9qQqxNcaHlnWkw4I7QyQQgItElOFnGxy9wj8PG5n7xPEGZw+jJFX
J9v1pa7t92z/HpxxMrVRpuRpUCqeXRCmter7AeHh+ClPxTjEcnq7McZAFHJ/32d/
k0JXL50d/FP4gxDEsnTOZoUXKHhPfT+QGieTKwIDAQABAoIBABXynEhDMxYdl7fD
RfIfCvosMsyw7AC3dx3/0exWgzBVvnnfifAQPVZ0+ibxDcRF6/A7qb9vKj3c5bP5
rKS323v2DsSGb07LAtDcr4WqW9RQO2kFfD64/kbPOBsSqSh7LH+lWkHetXEbnZ3G
97uZl63+fXGW7N2PC4ACXapnAKEOSqi+gBRTS9kaMUrnCkDTF60HVXgCJSQ7z5jI
5OVhD5Mbld6ey/FtZWIuNHRfdN1eQWDnPB5Twia/w6lAA7RiNdWmLMOPezOKgzt8
nZeriOzgphFSwKVerifXeN3bH8ZSCjfZeUJEULqJMlLNMFX0rTLE5DnvLsSKDhVZ
GTmnlMECgYEA8RY0+2Y+zdxPmNeT88tyFWFAlqFC1XzgfEV4RDdS30O1j/OWgLug
GMZZaPUqHcR5gbmR0kE7IgnihsT83YLlhw1UqlVNr0oHRdEiPALFSZWtXmEigmL5
ATpVkvzCPYLM2Y7GKC0KjqODAKT7vkhs3QiBGC24Isk6wnu4mZMD9b0CgYEAzfR9
jXxcbiOcMQxEtzYOt8RHK2qZueS2ZHPrpL+jz3VO8beWV0x8QsA+vkQQi6eZh43v
4X+f+0n9Mz52T9nsIwos7a30qecCzSewy8KSwWz6seqstM0lZ9yKwmV7ZI5FQFjJ
n1kLxUXkGIIqjYVOdnUiJCyIKqWh+32NFneldwcCgYBVu/n3dTLcbn538iIRTsYx
SDZ52rszxIboaqQMTkhQUXndyVg6S37xkIf3RUhG8bnsIhvzU3M99ZnzJ1ZtR8Fs
N5dmQ5OuxHq1n1u3T5ztQCc5Kv3ktGGsyBvqe6xvx3Yl4Boj63CXrRpHPmSwFH1y
VTXjbU7QQBN/WhQZuShtWQKBgFasGVo9h7hiDA0oGJLxkOgNFdZTmjX85jUIA5bl
f06DE6HtnLtIWNLOjJkNk9jxw06kkNrZFr5tuLykAK5CC3oDgjzUBC6sJSWpGmHl
lFUKQ8Yz+FJuMtuumEaO5oq+xK37r337v1RZwU//F2FPRGHdkWyfwP4RjkWD5Kgb
2eppAoGADtbLzvT4YGMGBSnW2fpROI1sy7xIhMyzyVlHj9Il79qGZIZF5mkzt6io
vVm9f4Jgssn1PPSsKuUSN8HjVYMn12/+eGQB0S1P1dgRSs+RIdbnSe0ZWvy9lup4
Ytgzg5pfy+8i/PuodwYg/5AfQrxziSzvdFgWiWlCfgm566005Ws=
-----END RSA PRIVATE KEY-----

[root@jenkins ~]# ssh-copy-id 10.211.55.201
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub"
The authenticity of host '10.211.55.201 (10.211.55.201)' can't be established.
ECDSA key fingerprint is SHA256:ZAzP64mdrYfOM1o3bwXlqY1JFOiNsbAbiHnWKv1ogLo.
ECDSA key fingerprint is MD5:d9:12:05:84:0e:79:7c:5f:6b:13:a2:97:6f:cb:a7:72.
Are you sure you want to continue connecting (yes/no)? yes
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
root@10.211.55.201's password: 

Number of key(s) added: 1

Now try logging into the machine, with:   "ssh '10.211.55.201'"
and check to make sure that only the key(s) you wanted were added.

# 验证
[root@jenkins ~]# ssh root@10.211.55.201
Last login: Sun Jan  7 13:03:41 2024 from 10.211.55.2
[root@master01 ~]#

3.5 Jenkins登录Harbor

# 修改docker配置文件,添加Harbor地址
vim /etc/docker/daemon.json

# 内容BEGIN
#insecure-registries表示后面的registry地址为非安全的.在master和node都要加,不然拉取不了镜像
"insecure-registries":["10.211.55.20"]
# 内容END

sudo systemctl daemon-reload && sudo systemctl restart docker

# 测试登录, harbor关闭防火墙 systemctl stop firewalld && systemctl disable firewalld
docker login harbor.zhinian.com -u jenkins -p Harbor@123456

遇到问题

解决

[root@jenkins ~]# docker login harbor.zhinian.com -u jenkins -p Harbor@123456
WARNING! Using --password via the CLI is insecure. Use --password-stdin.
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

3.6 Jnekins配置-继续

问题:控制台输出报错

Started by GitLab push by 王小虎
Running as SYSTEM
Building in workspace /var/lib/jenkins/workspace/springboot-demo
[springboot-demo] $ /bin/sh -xe /tmp/jenkins7820913899171498991.sh
+ pwd
/var/lib/jenkins/workspace/springboot-demo
+ ls
+ mvn clean package -Dmaven.test.skip=true
/tmp/jenkins7820913899171498991.sh:行4: mvn: 未找到命令
Build step 'Execute shell' marked build as failure
Finished: FAILURE

解决:

ln -s /opt/maven/apache-maven-3.8.6/bin/mvn /usr/bin/mvn

Build Steps

pwd
ls
git --version
rm -rf springboot-demo && git clone git@gitlab.zhinian.com:opit/springboot-demo.git
cd springboot-demo && mvn clean package -Dmaven.test.skip=true
docker login harbor.zhinian.com -u jenkins -p Harbor@123456
docker build -t springboot-demo:latest .
docker tag springboot-demo:latest harbor.zhinian.com/opit/springboot-demo:latest
docker push harbor.zhinian.com/opit/springboot-demo:latest && docker rmi springboot-demo:latest && docker rmi harbor.zhinian.com/opit/springboot-demo:latest
scp -r script/springboot-demo.yaml root@10.211.55.201:~/kube-conf/
ssh 10.211.55.201 'kubectl apply -f /root/kube-conf/springboot-demo.yaml'
ssh 10.211.55.201 'kubectl get pod -n prod'

3.7 提交代码进行测试

# 提交并Push到仓库
git commit && git push ... after

# 自动构建部署
1.jenkins auto build
2.maven build success
3.docker build success
4.docker push harbor success
5.kubectl apply success

[root@master01 kube-conf]# kubectl get po springboot-demo-deploy-c4c5cf4d8-bs6k7 -n prod
NAME                                     READY   STATUS    RESTARTS   AGE
springboot-demo-deploy-c4c5cf4d8-bs6k7   1/1     Running   1          6m7s

四、基于GitLab的CICD自动构建部署

Waiting...

• https://cloud.tencent.com/developer/article/1444269
• https://zhuanlan.zhihu.com/p/583345352
• https://blog.csdn.net/littlehaes/article/details/104694375

# https://hub.docker.com/_/openjdk/tags?page=6
docker pull openjdk:22-ea-jdk-oraclelinux8
docker tag openjdk:22-ea-jdk-oraclelinux8 java:8