Jenkins - Parallel 并行执行
- 引言
- [Jenkins Parallel 基本用法](#Jenkins Parallel 基本用法)
- [Jenkins Prallel 启动多个其他 Job](#Jenkins Prallel 启动多个其他 Job)
- [Jenkins Parallel Retry](#Jenkins Parallel Retry)
引言
日常执行测试用例，为了缩减执行时间，提高工作效率，可以从以下两个方面进行改进。
- 代码层面 ：用多进程并发执行代替单进程串行执行，更多细节请参考《Python 多线程和多进程并发执行》
- Jenkins 层面：我们可以将测试用例拆分成多个 Job，然后并行执行这些 Job。
今天主要了解一下 Jenkins 并行多个 Job 执行。
Jenkins Parallel 基本用法
在 Jenkins pipeline 中，parallel 关键字用于并行执行多个步骤。这可以显著减少构建时间，特别是当你有多个独立的任务需要完成时。
下面这个脚本定义了一个 Jenkins pipeline，其中包含一个 Parallel Jobs 阶段，使用 parallel 块来定义并行执行的多个子阶段，该阶段下有三个并行的子阶段（Job 1、Job 2 和 Job 3）。每个子阶段包含具体的步骤，例如 echo 和 sh 命令。
groovy
pipeline {
agent any
stages {
stage('Parallel Jobs') {
parallel {
stage('Job 1') {
steps {
echo 'Running Job 1'
// 在这里添加 Job 1 的具体执行步骤
sh 'echo "Job 1 executed"'
}
}
stage('Job 2') {
steps {
echo 'Running Job 2'
// 在这里添加 Job 2 的具体执行步骤
sh 'echo "Job 2 executed"'
}
}
stage('Job 3') {
steps {
echo 'Running Job 3'
// 在这里添加 Job 3 的具体执行步骤
sh 'echo "Job 3 executed"'
}
}
}
}
}
}我们来看一下运行效果：Job 1、Job 2 和 Job 3 是同进运行的
bash
17:24:14 [Pipeline] {
17:24:14 [Pipeline] stage
17:24:14 [Pipeline] { (Parallel Jobs)
17:24:14 [Pipeline] parallel
17:24:14 [Pipeline] { (Branch: Job 1)
17:24:14 [Pipeline] { (Branch: Job 2)
17:24:14 [Pipeline] { (Branch: Job 3)
17:24:14 [Pipeline] stage
17:24:14 [Pipeline] { (Job 1)
17:24:14 [Pipeline] stage
17:24:14 [Pipeline] { (Job 2)
17:24:14 [Pipeline] stage
17:24:14 [Pipeline] { (Job 3)
17:24:14 [Pipeline] echo
17:24:14 Running Job 1
17:24:14 [Pipeline] sh
17:24:14 [Pipeline] echo
17:24:14 Running Job 2
17:24:14 [Pipeline] sh
17:24:14 [Pipeline] echo
17:24:14 Running Job 3
17:24:14 [Pipeline] sh
17:24:14 + echo Job 1 executed
17:24:14 Job 1 executed
17:24:14 [Pipeline] }
17:24:14 + echo Job 2 executed
17:24:14 Job 2 executed
17:24:14 [Pipeline] // stage
17:24:14 [Pipeline] }
17:24:14 + echo Job 3 executed
17:24:14 Job 3 executed
17:24:14 [Pipeline] }
17:24:14 [Pipeline] // stage
17:24:14 [Pipeline] }
17:24:14 [Pipeline] }
17:24:15 [Pipeline] // stage
17:24:15 [Pipeline] }
17:24:15 [Pipeline] // parallel
17:24:15 [Pipeline] }
17:24:15 [Pipeline] // stage
17:24:15 [Pipeline] }
17:24:15 [Pipeline] // node
17:24:15 [Pipeline] End of Pipeline
17:24:15 Finished: SUCCESSJenkins Prallel 启动多个其他 Job
上在举例中 3 个 Job 只是执行一些基础脚本，现实测试过程中，是需要并行执行多个其它 Job 的，并且需要传参，获取执行结果等等。
下面定义一个 Jenkins pipeline，主 job 传参给子 job，并行执行三个子 job，每个子阶段使用 build 步骤触发其他 Jenkins job （Testing_1，Testing_2，'Testing_3），并使用 wait: false 参数使其异步执行，最后将子 job 的信息传给 report job，发邮件汇总测试结果。
propagate: false 表示即使这个 job 失败，也不会导致整个 pipeline 失败。如果你希望 job 失败时 pipeline 也失败，可以将其设置为 true。
groovy
#!groovy
pipeline {
options {
buildDiscarder(logRotator(numToKeepStr: '50', daysToKeepStr: '100'))
}
agent { label "ubuntu" }
parameters {
choice(name: 'branch', choices: ["main","feature"])
choice(name: "Environment", choices: ["cert", "dev", "prod"])
string(name: 'project_name', defaultValue: 'Cert XXX', description: 'example : Cert XXX, Prod XXX')
choice(name: "sheet_name", choices: ["sheet 1", "sheet 2", "sheet 3"])
choice(name: "action", choices: ["action 1","action 2", "action 3"])
string(name: 'email_list', defaultValue: 'aaa@bbb.com',description: 'example : aaa@bbb.com, bbb@bbb.com')
booleanParam(name: 'SAVE_TEST_RESULT_TO_HISTORY', defaultValue: true, description: 'Would you like to save test result to history?')
}
stages {
stage('Parallel Jobs') {
parallel {
stage('Testing 1') {
agent { label "ubuntu" }
steps {
script {
def job_1 = build job: 'Testing_1',
propagate: false,
wait: true,
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'Environment', value: "${params.Environment}"),
string(name: "sheet_name", value: "${params.sheet_name}"),
string(name: "action", value: "${params.action}")
]
job_1_project_name = job_1.getFullProjectName()
job_1_build_num = job_1.getNumber()
}
}
}
stage('Testing_2') {
agent { label "ubuntu" }
steps {
script {
def job_2 = build job: 'Testing_2',
propagate: false,
wait: true,
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'Environment', value: "${params.Environment}"),
string(name: "sheet_name", value: "${params.sheet_name}"),
string(name: "action", value: "${params.action}")
]
job_2_project_name = job_2.getFullProjectName()
job_2_build_num = job_2.getNumber()
}
}
}
stage('Testing_3') {
agent { label "ubuntu" }
steps {
script {
def job_3 = build job: 'Testing_3',
propagate: false,
wait: true,
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'Environment', value: "${params.Environment}"),
string(name: "sheet_name", value: "${params.sheet_name}"),
string(name: "action", value: "${params.action}")
]
job_3_project_name = job_3.getFullProjectName()
job_3_build_num = job_3.getNumber()
}
}
}
}
}
}
post {
always{
build job: 'Report',
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'project_name', value: "${params.project_name}"),
string(name: 'job_1_project_name', value: "${job_1_project_name}"),
string(name: 'job_1_build_num', value: "${job_1_build_num}"),
string(name: 'job_2_project_name', value: "${job_2_project_name}"),
string(name: 'job_2_build_num', value: "${job_2_build_num}"),
string(name: 'job_3_project_name', value: "${job_3_project_name}"),
string(name: 'job_3_build_num', value: "${job_3_build_num}"),
string(name: 'email_list', value: "${params.email_list}"),
booleanParam(name: 'SAVE_TEST_RESULT_TO_HISTORY', value: "${params.SAVE_TEST_RESULT_TO_HISTORY}")
]
}
}
}Jenkins Parallel Retry
上面例子的 Jenkins pipeline，并行执行多个 Job，也会有遇到一些异常，比如某一个子 job 因为环境问题导致失败，尽管不会中断整个 pipeline job，但是会影响整个 pipeline 的执行结果，只得到部分执行结果。文章《Jenkins - Pipeline Retry》只是简单的 job retry，对于并行执行其它 jenkins job 不太适合。
对于并行执行其它 jenkins job，retry 是必须要加上的，此外只要 job 执行失败必须得抛出 error 手动抛出异常，触发重试，不然不会 retry 重试。
所以将上面的 jenkins pipeline 脚本改一下，加上 retry ：
parallel 块：并行执行多个 job （Testing_1，Testing_2，Testing_3），每个 job 独立处理其失败和重试逻辑。
retry 步骤：retry(3) 表示在一个 job 失败时最多重试 3 次。
error 步骤：在捕获的异常中使用 error 步骤手动抛出异常，触发重试。
groovy
#!groovy
pipeline {
options {
buildDiscarder(logRotator(numToKeepStr: '50', daysToKeepStr: '100'))
}
agent { label "ubuntu" }
parameters {
choice(name: 'branch', choices: ["main","feature"])
choice(name: "Environment", choices: ["cert", "dev", "prod"])
string(name: 'project_name', defaultValue: 'Cert XXX', description: 'example : Cert XXX, Prod XXX')
choice(name: "sheet_name", choices: ["sheet 1", "sheet 2", "sheet 3"])
choice(name: "action", choices: ["action 1","action 2", "action 3"])
string(name: 'email_list', defaultValue: 'aaa@bbb.com',description: 'example : aaa@bbb.com, bbb@bbb.com')
booleanParam(name: 'SAVE_TEST_RESULT_TO_HISTORY', defaultValue: true, description: 'Would you like to save test result to history?')
}
stages {
stage('Parallel Jobs') {
parallel {
stage('Testing 1') {
agent { label "ubuntu" }
steps {
retry(3) {
script {
def job_1 = build job: 'Testing_1',
propagate: false,
wait: true,
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'Environment', value: "${params.Environment}"),
string(name: "sheet_name", value: "${params.sheet_name}"),
string(name: "action", value: "${params.action}")
]
job_1_project_name = job_1.getFullProjectName()
job_1_build_num = job_1.getNumber()
if (job_1.result != 'SUCCESS') {
error "Job Content_Testing_1 failed with result: ${job_1.result}"
}
}
}
}
}
stage('Testing_2') {
agent { label "ubuntu" }
steps {
retry(3) {
script {
def job_2 = build job: 'Testing_2',
propagate: false,
wait: true,
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'Environment', value: "${params.Environment}"),
string(name: "sheet_name", value: "${params.sheet_name}"),
string(name: "action", value: "${params.action}")
]
job_2_project_name = job_2.getFullProjectName()
job_2_build_num = job_2.getNumber()
if (job_2.result != 'SUCCESS') {
error "Job Testing_2 failed with result: ${job_2.result}"
}
}
}
}
}
stage('Testing_3') {
agent { label "ubuntu" }
steps {
retry(3) {
script {
def job_3 = build job: 'Testing_3',
propagate: false,
wait: true,
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'Environment', value: "${params.Environment}"),
string(name: "sheet_name", value: "${params.sheet_name}"),
string(name: "action", value: "${params.action}")
]
job_3_project_name = job_3.getFullProjectName()
job_3_build_num = job_3.getNumber()
if (job_3.result != 'SUCCESS') {
error "Job Testing_3 failed with result: ${job_3.result}"
}
}
}
}
}
}
}
}
post {
always{
build job: 'Report',
parameters: [
string(name: 'branch', value: "${params.branch}"),
string(name: 'project_name', value: "${params.project_name}"),
string(name: 'job_1_project_name', value: "${job_1_project_name}"),
string(name: 'job_1_build_num', value: "${job_1_build_num}"),
string(name: 'job_2_project_name', value: "${job_2_project_name}"),
string(name: 'job_2_build_num', value: "${job_2_build_num}"),
string(name: 'job_3_project_name', value: "${job_3_project_name}"),
string(name: 'job_3_build_num', value: "${job_3_build_num}"),
string(name: 'email_list', value: "${params.email_list}"),
booleanParam(name: 'SAVE_TEST_RESULT_TO_HISTORY', value: "${params.SAVE_TEST_RESULT_TO_HISTORY}")
]
}
}
}