k8s部署自动化工具jenkins

一、项目背景

使用ack或者自建k8s集群部署一套自动化部署工具jenkins，实现代码快速构建部署上线

二、部署安装

使用了ack的卷，需要关注一下

bash 复制代码

#apiVersion: v1
#kind: PersistentVolumeClaim
#metadata:
#  name: jenkins-pvc
#  namespace: jenkins
#spec:
#  accessModes:
#    - ReadWriteOnce
#  resources:
#    requests:
#      storage: 500Gi
#  storageClassName: alibabacloud-cnfs-nas
#---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins
  namespace: jenkins
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins
  namespace: jenkins
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: jenkins
  namespace: jenkins
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins
subjects:
- kind: ServiceAccount
  name: jenkins
  namespace: jenkins
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: jenkins
  namespace: jenkins
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      containers:
      - name: jenkins
        #image: mirror-registry.cn-hangzhou.cr.aliyuncs.com/hzdx/prod/jenkins:2.462.3-lts-jdk21
        image: registry.cn-hangzhou.aliyuncs.com/tym-test/demo:jenkins
        ports:
        - containerPort: 8080
        - containerPort: 50000
        env:
        - name: TZ
          value: Asia/Shanghai
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              divisor: 1Mi
              resource: limits.memory
        - name: JAVA_OPTS
          value: -Djenkins.model.Jenkins.slaveAgentPort=50000 -Dhudson.lifecycle=hudson.lifecycle.ExitLifecycle
            -Duser.timezone=Asia/Shanghai -Djava.awt.headless=true -Dorgjenkinsci.plugins.gitclient.Git.timeOut=30
        livenessProbe:
          failureThreshold: 5
          httpGet:
            path: /login
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        ports:
        - containerPort: 8080
          name: httpport
          protocol: TCP
        - containerPort: 50000
          name: jnlpport
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /login
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        securityContext:
          privileged: true
          runAsUser: 0
        volumeMounts:
        - mountPath: /var/jenkins_home
          name: jenkinshome-pvc
        - mountPath: /run/containerd/containerd.sock
          name: containerd-sock
        - mountPath: /usr/bin/ctr
          name: ctr-command
        - name: timezone
          mountPath: /etc/localtime
          readOnly: true
      serviceAccountName: jenkins
      volumes:
      #- name: jenkinshome
        #persistentVolumeClaim:
        #  claimName: jenkins-pvc
      - hostPath:
          path: /run/containerd/containerd.sock
          type: ""
        name: containerd-sock
      - hostPath:
          path: /usr/bin/ctr
          type: ""
        name: ctr-command
      - name: timezone
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
  volumeClaimTemplates:
  - metadata:
      name: jenkinshome-pvc
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "cnfs-nas-sc" # 引用名称为cnfs-nas-sc的StorageClass对象。
      resources:
        requests:
          storage: 500Gi
---
# jenkins-service.yaml
apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/port: "8080"
    prometheus.io/scrape: "true"
  labels:
    app: jenkins
  name: jenkins
  namespace: jenkins
spec:
  ports:
  - name: web
    nodePort: 30880
    port: 8080
    protocol: TCP
    targetPort: 8080
  - name: agent
    nodePort: 30500
    port: 50000
    protocol: TCP
    targetPort: 50000
  selector:
    app: jenkins
  type: NodePort

四、配置ingress的方式进行访问

密码查看：

bash 复制代码

kubectl logs <jenkins-pod-name> | grep "Administrator password"

五、编写流水线脚本

示例：（根据实际情况进行编写，所用镜像需要替换）

bash 复制代码

currentBuild.displayName = "${JOB_NAME}" + "项目第" + "${BUILD_NUMBER}" + "次构建"
currentBuild.description =   "${JOB_NAME}" + "job"

def now = new Date().format('yyyy-MM-dd HH:mm:ss', TimeZone.getTimeZone('Asia/Shanghai'))
def timestamp = new Date().format('yyyyMMddHHmmss')

pipeline {
  agent {
    kubernetes {
      yaml '''
        apiVersion: v1
        kind: Pod
        metadata:
          labels:
            k8s-app: jenkins-agent
        spec:
          volumes:
          - name: "maven-conf"
            configMap:
              name: "maven-config"  # 使用已存在的 ConfigMap
              items:
              - key: "settings.xml"   # ConfigMap 中的 key
                path: "settings.xml" # 挂载后的文件名
          - name: "tz"
            hostPath:
              path: "/etc/localtime"
              type: ""
          - name: "ssh"
            hostPath:
              path: "/root/.ssh"
          - name: "m2"
            hostPath:
              path: "/root/.m2"
          containers:
          - name: git
            image: mirror-registry-vpc.cn-hangzhou.cr.aliyuncs.com/test/alpine-git:v2.49.1
            command:
            - cat
            tty: true
          - name: jnlp
            image: mirror-registry-vpc.cn-hangzhou.cr.aliyuncs.com/test/jenkins-inbound-agent:3309.v27b_9314fd1a_4-1
            #args: ['\$(JENKINS_SECRET)', '\$(JENKINS_NAME)']
          - name: maven
            image: mirror-registry-vpc.cn-hangzhou.cr.aliyuncs.com/test/maven:3.9.9-eclipse-temurin-8-alpine
            command:
            - cat
            tty: true
            volumeMounts:
            - mountPath: "/root/.m2"
              name: "m2"
            - mountPath: "/usr/share/maven/conf/settings.xml"
              name: "maven-conf"
              subPath: "settings.xml"
            - mountPath: "/root/.ssh"
              name: "ssh"
            - name: "tz"
              mountPath: "/etc/localtime"
          - name: kaniko
            image: mirror-registry-vpc.cn-hangzhou.cr.aliyuncs.com/test/kaniko:1.25.0-configjson
            command:
            - cat
            tty: true
            volumeMounts:
            - name: "tz"
              mountPath: "/etc/localtime"
              readOnly: true
        '''
      retries 2
    }
  }

    environment {
        gitUrl = "xxxx"
        registryVpc = "mirror-registry-vpc.cn-hangzhou.cr.aliyuncs.com"
        registryCredential = "Ali-ACR-credential"
        ImageUrl = "${registryVpc}/test"
        dest_path = "/var/jenkins_home/workspace/${JOB_NAME}"
        job_path = "/data/docker-compose/jenkins/jenkins_home/workspace/${JOB_NAME}"
        JAVA_OPTS= "-Djava.security.egd=file:/dev/./urandom -Dfile.encoding=utf-8"
        yml_dir = "/opt/yaml"
        remote_host = "xxxxx"
    }

  	options {
        disableConcurrentBuilds()
        timestamps()
	  }

    parameters {
        choice(name: 'mode', choices: ['deploy','rollback'], description: '请选择发布或者回滚')
        gitParameter(name: 'BRANCH_NAME', type: 'PT_BRANCH_TAG', branchFilter: 'origin/(.*)', defaultValue: 'test', selectedValue: 'DEFAULT', sortMode: 'DESCENDING_SMART', description: 'Select your branch or tag.')
        extendedChoice(name: 'ProjectName',
                       type: 'PT_CHECKBOX',
                       description: '请勾选所要发布的项目模块',
                       quoteValue: false,
                       saveJSONParameterToFile: false,
                       value: 'test',
                       visibleItemCount: 10,
                       multiSelectDelimiter: ',',
                       defaultValue: 'test'
		    )
    }


    stages {
        stage('初始化信息') {
            when {
                environment name: 'mode', value: 'deploy'
            }
            steps {
                container('maven') {
                    script {
                        // 获取并处理分支名称（移除origin/前缀）
                        def branchName = params.BRANCH_NAME
                        if (branchName.startsWith('origin/')) {
                            branchName = branchName.replace('origin/', '')
                        }
                        env.CURRENT_BRANCH = branchName
                        
                        echo "=============================================="
                        echo "🎯 开始构建项目: ${JOB_NAME}"
                        echo "📦 构建编号: ${BUILD_NUMBER}"
                        echo "🌿 当前构建分支: ${env.CURRENT_BRANCH}"
                        echo "📅 构建时间: ${now}"
                        echo "🔧 构建模式: ${params.mode}"
                        echo "📋 选择项目: ${params.ProjectName}"
                        echo "=============================================="
                        
                        // 更新构建描述信息，包含分支信息
                        currentBuild.description = "${JOB_NAME}job - 分支: ${env.CURRENT_BRANCH}"
                    }
                }
            }
        }

        stage('CheckOut') {
            when {
                environment name: 'mode', value: 'deploy'
            }
            steps {
                container('maven') {
                    script {
                        echo "📥 正在克隆代码库，分支: ${env.CURRENT_BRANCH}"
                        sh """
                            ssh ${remote_host} -o StrictHostKeyChecking=no "rm -rf /opt/test && cd /opt && pwd && git clone -b ${env.CURRENT_BRANCH} ${gitUrl}"
                        """
                        echo "✅ 代码克隆完成，分支: ${env.CURRENT_BRANCH}"
                    }
                }
            }
        }

      stage('Run maven') {
          when {
              environment name: 'mode',value: 'deploy'
          }
          steps {
              container('maven') {
                script {
                    echo "🔨 开始Maven构建，分支: ${env.CURRENT_BRANCH}"
                    sh """
                        ssh ${remote_host} -o StrictHostKeyChecking=no "source /etc/profile && cd /opt/test && mvn -DskipTests clean package"
                    """
                    echo "✅ Maven构建完成，分支: ${env.CURRENT_BRANCH}"
                }
              }
          }
      }

        stage('Image build') {
            when {
                environment name: 'mode',value: 'deploy'
            }
            steps {
                container('maven') {
                    script {
                        env.currentDate = sh (script: 'date +%Y-%m-%d_%H-%M-%S', returnStdout: true).trim()
                        def selectedProjects = params.ProjectName.split(',')
                        
                        echo "🐳 开始构建Docker镜像，分支: ${env.CURRENT_BRANCH}"
                        echo "📦 镜像标签时间戳: ${env.currentDate}"
                        
                        // 获取用户选择的项目模块
					    for (def project : selectedProjects) {
					        if (project.trim()) {
                                echo "🔧 正在构建项目: ${project}，分支: ${env.CURRENT_BRANCH}"
                                
                                sh """
                                    ssh ${remote_host} -o StrictHostKeyChecking=no "
                                        cd /opt/test/mid-holo-biz/target/
                                        ls -lh *.jar
        cat << EOF > Dockerfile
FROM mirror-registry-vpc.cn-hangzhou.cr.aliyuncs.com/test/eclipse-temurin:jdk-8u452

ENV TZ=Asia/Shanghai
ENV LANG C.UTF-8
RUN mkdir /app/ -p
COPY *.jar /app/${project}.jar
ENTRYPOINT [\\"sh\\", \\"-c\\", \\"java -Djava.security.egd=file:/dev/./urandom -Dfile.encoding=utf-8 -jar /app/${project}.jar\\"]
EOF

                                        cat Dockerfile
										docker build . -t ${ImageUrl}/${project}:${currentDate}
										docker push ${ImageUrl}/${project}:${currentDate}"
                                """
                                echo "✅ 项目 ${project} 镜像构建完成"
                            }
                        }
                    }
                }
            }
        }

        stage('Update') {
            when {
                environment name: 'mode',value: 'deploy'
            }
            steps {
                container('maven') {
					script {
                        def selectedProjects = params.ProjectName.split(',')
                        
                        echo "🔄 开始更新部署，分支: ${env.CURRENT_BRANCH}"
                        
                        // 获取用户选择的项目模块
					    for (def project : selectedProjects) {
					        if (project.trim()) {  // 添加非空检查
                                echo "🔄 正在更新项目: ${project}，分支: ${env.CURRENT_BRANCH}"
                                
							    sh """
                                    ssh ${remote_host} -o stricthostkeychecking=no "sh /opt/demo/dxtest-charts/all.sh Change=deploy   Name=mid-holo Image=${ImageURL}/${project}:${currentDate} namespace=test"
								    """
                                
                                echo "✅ 项目 ${project} 更新完成"
						    }
					    }
				    }
			    }
		    }
	    }

        stage('部署完成') {
            when {
                environment name: 'mode', value: 'deploy'
            }
            steps {
                script {
                    echo "=============================================="
                    echo "🎉 部署完成!"
                    echo "📦 项目: ${JOB_NAME}"
                    echo "🌿 分支: ${env.CURRENT_BRANCH}"
                    echo "🔢 构建编号: ${BUILD_NUMBER}"
                    echo "📅 完成时间: ${new Date().format('yyyy-MM-dd HH:mm:ss', TimeZone.getTimeZone('Asia/Shanghai'))}"
                    echo "🐳 镜像标签: ${env.currentDate}"
                    echo "=============================================="
                }
            }
        }

    }
}

六、部署结果查看