Jenkins - 技术栈

一. Jenkins

1. 部署

bash 复制代码

# 解决报错: 无法验证 pkg.jenkins.io 的由 "/C=US/O=Let's Encrypt/CN=R3" 颁发的证书: 颁发的证书已经过期。 要以不安全的方式连接至 pkg.jenkins
yum install -y ca-certificates

wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo
rpm --import https://pkg.jenkins.io/redhat/jenkins.io.key

yum install fontconfig java-11-openjdk -y
yum install jenkins -y

或者这样

bash 复制代码

cat >>/etc/yum.repos.d/jenkins.repo<<EOF
[jenkins]
name=Jenkins-stable
baseurl=http://pkg.jenkins.io/redhat-stable
gpgcheck=1
EOF

bash 复制代码

rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key
yum install java-11-openjdk -y
yum install jenkins -y 
systemctl daemon-reload
systemctl enable jenkins;systemctl start jenkins

2. 插件

bash 复制代码

git
Chinese
pipeline
GitLab
Maven
Git Parameter				# git 参数化构建
Extended Choice Parameter	# 选项参数，可多选
Pipeline: Stage View		# 阶段视图
blue ocean					# 流水线视图
nodejs						# 前端打包npm

# workspace清理插件
ws-cleanup

# 连接k8s
Kubernetes CLI

# 角色控制
Role-based Authorization Strategy

# 钉钉消息
dingding

1). 离线安装插件

官网插件下载

阿里云插件下载

先下载插件到本地，然后如下，Advanced settings ==> Deploy Plugin ==> 上传

3. Manage and Assign Roles

权限管控
安装这个插件后, 需要在全局安全设置中设置, 才能打开, 旧版本可以不用设置
- 系统管理==>全局安全设置

在授权策略中设置为: Role-based Strategy, 应用保存

之后返回系统管理界面, 可以看到Manage and Assign Roles

1). Manage Roles

Global roles
- 全局角色
Item roles
- 项目角色
Node roles
- 代理角色
需要先配置Manage Roles , 然后配置Assign Roles

2). 使用案例

创建zhangsan 和 lisi 两个用户

创建2个任务

设置 Manage Roles , 只需要给 Read 权限即可

点击 Assign Roles 配置

4. Kubernetes CLI

Kubernetes CLI | Jenkins

首先要在Jenkins上安装插件 Kubernetes CLI ,然后在Kubernetes上创建用户

bash 复制代码

[root@master-3 ~/images]# kubectl -n <namespace> create serviceaccount jenkins-robot

# 如果想给单独 Namespace 创建只需要创建 rolebinding， 如果是整个集群，就使用 clusterrolebinding
[root@master-3 ~/images]# kubectl -n <namespace> create clusterrolebinding jenkins-robot-binding --clusterrole=cluster-admin --serviceaccount=<namespace>:jenkins-robot

# 获得 sa的 secret
[root@master-3 ~/images]# kubectl -n <namespace> get serviceaccount jenkins-robot -o go-template --template='{{range .secrets}}{{.name}}{{"\n"}}{{end}}'

[root@master-3 ~/images]# kubectl -n hhy get secrets `kubectl -n hhy  get sa jenkins-robot -o go-template --template='{{range .secrets}}{{.name}}{{"\n"}}{{end}}'` -o go-template --template '{{index .data "token"}}' | base64 -d

然后再 Jenkins 上创建 Secret text 类型的凭据，将上面 base64 -d 获得的信息贴进去，就可以在 Jenkins 上使用 withKubeConfig() 连接集群
测试该角色是否有权限，随便找个名称空间下，看是否可以获得他们的信息

sh 复制代码

curl -k  -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkcyTjNrRFYwNGZmNl9Qc203N2YySUllNUg2T3lTSWg2cm9FUDdiUWdZbnMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJqZW5raW5zLXJvYm90LXRva2Vxxxxxxxxxxxxxx" https://192.99.99.99:6443/api/v1/namespaces/ingress-nginx/pods

Jenkins在集群中可以这样写

sh 复制代码

container('kubectl') {
    withKubeConfig([credentialsId: "Kubernetes Token 凭据 ID",serverUrl: "https://kubernetes.default.svc.cluster.local"]) {
        sh "kubectl get nodes"
    }
}

5. jenkins SSH拉取代码

sh 复制代码

# jenkin 上创建秘钥对
ssh-keygen

# 复制到 gitlab 上， https://gitlab.yee.net.cn/-/profile/keys
cat ~/.ssh/id_rsa.pub

# Jenkins 上创建 "SSH Username with private key" 类型的 credentials（凭据）

# 测试是否通
ssh -T git@gitlab.yee.net.cn

安全方面提示：
sh 复制代码
You're using 'Known hosts file' strategy to verify ssh host keys, but your known_hosts file does not exist, please go to 'Manage Jenkins' -> 'Security' -> 'Git Host Key Verification Configuration' and configure host key verification.
Manage Jenkins ==> Security ==> Git Host Key Verification Configuration

凭据创建

报错：
sh 复制代码
Couldn't find any revision to build. Verify the repository and branch configuration for this job
因为最新的 gitlab 的默认分支不再是master，而是main分支

可以在 Gitlab上修改默认分支为 master

sh 复制代码

pipeline {
    agent any
    environment {
        GIT_PATH = "git@xxxx.git"
        PROJECT_NAME = "${JOB_NAME}"
        IMAGE_HOST_TEST = "harbor.yee.net.cn"
        MVN_CMD = "mvn clean -U install -Pprd -Dmaven.test.skip=true"
    }

    tools {
        maven "maven-3.3.9"
    }

    stages {
        stage('Init') {
            steps {
                script{
                    images = []
                   _imageTag = ""
                   _imageGroup = env.IMAGE_GROUP
                   _dockerFile = "dockerfile"
                   _deployServiceNames = []
                   _deployEnv = params.deploy_env //test
                   _isDev = false
                   _canDeploy = true
                   _serviceNames = []
                   if(params.service_types!=""){
                       for (type in params.service_types.tokenize(',')){
                            _serviceNames.add("${PROJECT_NAME}-${type}")
                       }
                   }else{
                       echo "[Error]没有选择服务"
                   }
                   
                   if(_imageGroup==null || _imageGroup==""){
                       _imageGroup = "${PROJECT_NAME.split('-')[0]}"
                   }
                   
                   _imageHost = "${IMAGE_HOST_TEST}"
                   echo "发布环境: ${_deployEnv}"
                   
                  if(params.branch_or_tag == "" || _serviceNames.size()==0){
                      _canDeploy = false
                  }
                }
            }
        }
        stage('Checkout') {
            steps {
                git branch: 'main', credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"
                

                # 这种方法也可以
                // // https://plugins.jenkins.io/git-parameter/
                // checkout([$class: 'GitSCM', 
                //     branches: [[name: "${branch_or_tag}"]], 
                //     extensions: [], 
                //     userRemoteConfigs: [[credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"]]
                // ])
            }
        }
    }
}

6. maven

maven3.39版本下载一

maven3.39版本下载二

sh 复制代码

# 插件目录
/var/lib/jenkins/plugin

7. 案例

使用 Username with password 类型的凭据

sh 复制代码

    environment {
        // HARBOR="harbor.devopsing.site"
        HARBOR_ACCESS_KEY = credentials('harbor-userpwd-pair')
            }
    .....

    docker login --username=${HARBOR_ACCESS_KEY_USR} --password=${HARBOR_ACCESS_KEY_PSW} ${HARBOR}

JAVA

sh 复制代码

pipeline {
    agent any
    environment {
        GIT_PATH = "git@xxx.git"
        HARBOR_PRJ = "yee"
        DOCKER_FILE = "dockerfile"        
        PROJECT_NAME = "${JOB_NAME}"
        IMAGE_HOST_TEST = "harbor.yee.cn"
        MVN_CMD = "mvn clean -U install -Pprd -Dmaven.test.skip=true"
    }

    tools {
        maven "maven-3.3.9"
    }

    stages {
        stage('Init') {
            steps {
                script{
                    images = []
                   _imageTag = ""
                   _imageGroup = env.IMAGE_GROUP
                   _deployServiceNames = []
                   _deployEnv = params.deploy_env //test
                   _isDev = false
                   _canDeploy = true
                   _serviceNames = []
                   if(params.service_types!=""){
                       for (type in params.service_types.tokenize(',')){
                            _serviceNames.add("${type}")
                       }
                   }else{
                       echo "[Error]没有选择服务"
                   }
                   
                   if(_imageGroup==null || _imageGroup==""){
                       _imageGroup = "${PROJECT_NAME.split('-')[0]}"
                   }
                   
                   _imageHost = "${IMAGE_HOST_TEST}"
                   echo "发布环境: ${_deployEnv}"
                   
                  if(params.branch_or_tag == "" || _serviceNames.size()==0){
                      _canDeploy = false
                  }
                }
            }
        }
        stage('Checkout') {
            steps {
                // git branch: 'main', credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"
                
                echo "git: ${GIT_PATH}"
                echo "branch: ${branch_or_tag}"
                
                https://plugins.jenkins.io/git-parameter/
                checkout([$class: 'GitSCM', 
                    branches: [[name: "${branch_or_tag}"]], 
                    extensions: [], 
                    userRemoteConfigs: [[credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"]]
                ])
            }
        }
        stage(' maven ') {
            when { expression { _canDeploy } }
            steps {
                script{
                    sh "${MVN_CMD}"
                }
            }
        }
        stage( ' deploy ' ) {
	       steps {
	           script{
	               def imageTag = sh(script:"date '+%Y%m%d%H%M%S'", returnStdout: true).replace("\n","")
                   for (serviceName in _serviceNames){
                        def workDic = "${WORKSPACE}/${PROJECT_NAME}-${serviceName}"
                        HARBOR_IMAGE_PUSH = "${IMAGE_HOST_TEST}/${HARBOR_PRJ}/${PROJECT_NAME}-${serviceName}:${imageTag}"
                        def CONFIG_FILE = "/data/k8s-deploy-scripts/test-${PROJECT_NAME}-${serviceName}.yaml"
                        
                        dir("${workDic}/target/"){
                            sh "docker build . -f ${JENKINS_HOME}/${DOCKER_FILE} -t ${HARBOR_IMAGE_PUSH}"
                            sh "docker push ${HARBOR_IMAGE_PUSH}"
                            sh "docker rmi ${HARBOR_IMAGE_PUSH}"                            
                        }
                        _deployServiceNames.add("${PROJECT_NAME}-${serviceName}");
                        images.add("${HARBOR_IMAGE_PUSH}")
                        
                        sh """
	                      ssh root@192.168.99.99 'yq -i ".spec.template.spec.containers[].image = \\"${HARBOR_IMAGE_PUSH}\\"" ${CONFIG_FILE}'
	                    """
	                    sh(script: "ssh root@192.168.99.99 kubectl apply -f ${CONFIG_FILE}")
                   }
	           }
	       }
        }
        
        stage(' output ') {
            steps {
                script{
                    echo "服务名称：${_deployServiceNames}"
                    echo "镜像：${images}"
                }
            }
        }         
    }
}

前端项目

sh 复制代码

//前端项目打包构建；支持多环境
pipeline {
    agent any
    environment {
        GIT_PATH = "git@xxx.git"
        DOCKER_FILE = "dockerfile"
        HARBOR_PRJ = "yee"
        IMAGE_HOST_TEST = "harbor.yee.cn"
        PROJECT_NAME = "${JOB_NAME}"
    }

    stages {
        stage('Init') {
            steps {
                script{
                    dev_imgFile = ""
                    prd_imgFile = ""
                   _imageTagForTest = ""
                   _imageTag = ""
                   _imageGroup = env.IMAGE_GROUP
                   _envs = ["prod","test"]
                
                   _envTexts = [prod:'生产环境',test:'测试环境']
                   
                   //if(!params.build_prd){
                   //   _envs = ["test"]
                   //}
                   
                   if(_imageGroup==null || _imageGroup==""){
                       _imageGroup = "${PROJECT_NAME.split('-')[0]}"
                   }
                }
            }
        }
        stage('Checkout') {
            steps {
                echo "git: ${GIT_PATH}"
                echo "branche: ${branch_or_tag}"
                
                checkout([$class: 'GitSCM', 
                    branches: [[name: "${branch_or_tag}"]], 
                    extensions: [], 
                    userRemoteConfigs: [[credentialsId: 'gitlab-ssh', url: "${GIT_PATH}"]]
                ])
                
                echo "code checkout completed"
            }
        }
        stage(' npm ') {
            when { expression {  params.branch_or_tag != "" } }
            steps {
                script{
                    def imageTag = sh(script:"date '+%Y%m%d%H%M%S'", returnStdout: true).replace("\n","")
                    def workDic = "${WORKSPACE}"
                    def imgName = "/${_imageGroup}/${PROJECT_NAME}"
                    nodejs('node14.17.3') {
                        sh "node -v"
                        sh "npm -v"
                        if(params.clean_cache){
                            sh "rm -rf package-lock.json"
                            sh "rm -rf node_modules/"
                        }
                        sh "npm install --unsafe-perm --legacy-peer-deps --registry https://registry.npmmirror.com"   
                        sh "npm run build:test"
                        dir("${workDic}/dist"){
                            HARBOR_IMAGE_PUSH = "${IMAGE_HOST_TEST}/${HARBOR_PRJ}/${PROJECT_NAME}:${imageTag}"
                            sh "docker build . -f ${JENKINS_HOME}/${DOCKER_FILE} -t ${HARBOR_IMAGE_PUSH}"
                            sh "docker push ${HARBOR_IMAGE_PUSH}"
                            sh "docker rmi ${HARBOR_IMAGE_PUSH}"                            
                        }
                    }

                }
            }
        }
        stage( ' deploy ' ) {
	       steps {
	           script{
	               def CONFIG_FILE = "/data/k8s-deploy-scripts/test-${PROJECT_NAME}.yaml"
	               
                   sh """
	                  ssh root@192.168.99.99 'yq -i ".spec.template.spec.containers[].image = \\"${HARBOR_IMAGE_PUSH}\\"" ${CONFIG_FILE}'
	               """
	               
	               sh(script: "ssh root@192.168.99.99 kubectl apply -f ${CONFIG_FILE}")
	           }
	       }
        }
        
        stage(' output ') {
            steps {
                script{
                    echo "服务名称：${PROJECT_NAME}"
                    echo "镜像：${HARBOR_IMAGE_PUSH}"
                }
            }
        }
    }
}

8. 登录 Jenkin 用户

sh 复制代码

su -s /bin/bash jenkins

9 参数

清除旧的构建

参数使用，一次性多选服务

10. slave

sh 复制代码

# 类似这种报错， 需要勾选： Use WebSocket
java.io.IOException: http://192.168.99.185:8000/ provided port:50000 is not reachable

二. 容器部署

yaml 复制代码

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: argocd
spec:
  replicas: 1
  selector:
    matchLabels:
      devops: jenkins
  template:
    metadata:
      labels:
        devops: jenkins
    spec:
      nodeName: k8s-208092-prd
      hostAliases:
       - ip: "xx.xx.xx.xx"
         hostnames:
         - "harbor.yee.net.cn"
      serviceAccount: jenkins
      initContainers:
      - name: fix-permissions
        image: busybox
        imagePullPolicy: IfNotPresent
        command:
        - /bin/sh
        - -c
        - |
          chown -R 1000:1000 /var/jenkins_home
        securityContext:
          privileged: true
        volumeMounts:
        - name: jenkinshome
          mountPath: /var/jenkins_home
      containers:
      - name: jenkins
        #image: jenkinsci/blueocean:1.25.5
        image: harbor.yee.com:8443/library/jenkins:1.25.5
        imagePullPolicy: IfNotPresent
        ports:
        - name: http
          containerPort: 8080
        - name: slavelistener
          containerPort: 50000
        volumeMounts:
        - name: jenkinshome
          mountPath: /var/jenkins_home
        - name: date-config
          mountPath: /etc/localtime
        - name: deploy-sh
          mountPath: /manifest 
        env:
        - name: JAVA_OPTS
          value: "-Xms4g -Xmx8g -Duser.timezone=Asia/Shanghai -Dhudson.model.DirectoryBrowserSupport.CSP="
      volumes:
      - name: jenkinshome
        persistentVolumeClaim:
          claimName: jenkins-pvc
      - name: date-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
      - name: deploy-sh
        configMap:
          name: jenkins-deploy-sh

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins-vie
  namespace: argocd
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  - name: slavelistener
    port: 50000
    targetPort: 50000
  type: ClusterIP
  selector:
    devops: jenkins
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jenkins
  namespace: argocd
  annotations:
    kubernetes.io/ingress.class: nginx
spec:
  rules:
  - host: jenkins-vie.yee.com
    http:
      paths:
      - backend:
          service:
            name: jenkins-vie
            port:
              number: 8080
        path: /
        pathType: Prefix
  tls:
  - hosts:
    - jenkins-ops.yee.com
    secretName: yee.com