rpmbuild多进程批量编译脚本

脚本用法

使用手册

bash 复制代码

bash spec.sh --help

bash 复制代码

Please ensure the directory ~/rpmbuild exists, And the script can execute in any directory.

Usage: bash spec.sh [OPTION]... [*.spec]... [OPTION]...
  or:  bash spec.sh [*.spec]... [OPTION]... [*.spec]...
rpmbuild all specfiles in the arguments, if it's empty, rpmbuild all specfiles in the SPECS directory

Arguments description
  -f, --force         skip existence check, force rpmbuild
  --thread=NUM        set the maximum number of executing processes
  --result_fix=char   set the output result symbol
  -* or --*           support all options in the rpmbuild

强制编译

编译会存在已编译检查，脚本已编译检查只写了简易校验(无语法解析)，校验准确度一般，校验不准确通过 -f触发强制编译即可。

bash 复制代码

bash spec.sh SPECS/qt.spec -f

设置执行线程数量

修改脚本代码中默认值或通过命令行参数 --thread动态修改。

bash 复制代码

bash spec.sh --thread=3

设置输出结果标记

修改脚本代码中默认值或通过命令行参数 --result_fix动态修改。

bash 复制代码

bash spec.sh --result_fix=#

修改rpmbuild单位宏定义

按需增删。

编译日志文件

log_file: 每次执行编译日志；log_file_history: 所有历史执行编译日志。

编译成功二进制包保存目录

其它

脚本代码其余参数范围（4 - 65）行有兴趣按需调整。

脚本代码

bash 复制代码

#!/bin/bash
script_start_time=$(date +%s)

# 日志文件
log_file=~/rpmbuild/compiled.log
log_file_history=~/rpmbuild/compiled.log.history
# 日志开关
log_switch=true
log_history_switch=true
# 执行最大进程数
thread_max_num=8
# 自定义宏,按公司需求定义
custom_define_arr=(
  --define='dist .test'
  --define='packager ****'
  --define='host ****'
  --define='vendor ****'
  --define='distribution ****'
)
# 编译完成rpm保存目录
success_dir=~/rpmbuild/success
# 编译完成rpm本地源目录
rpm_source=~/rpmbuild/RPMS/source
# dnf配置文件
dnf_config=/etc/dnf/dnf.conf
# rpmbuild参数
rpmbdf='-bb'
rpmbds=''
declare -a rpmbdo
# 管道文件绑定
lock_switch=false
lock_file=~/.spec.lock
lock_fd=''
builddep_lock_file=~/.builddep_spec.lock
builddep_lock_fd=''
global_lock_file=~/.global_spec.lock
global_lock_fd=''
# spec文件统计
declare -a arr
spec_count_file=~/.spec_count_file.log
spec_count=0
success_spec_count=0
already_spec_count=0
error_spec_count=0
missing_spec_count=0
declare -a success_spec_count_arr
declare -a already_spec_count_arr
declare -a error_spec_count_arr
declare -a missing_spec_count_arr
# 编译退出码
SUCCESS=1
ALREADY=2
MISSING=3
ERROR=4
SUCCESS_MESSAGE='编译成功'
ALREADY_MESSAGE='已编译'
MISSING_MESSAGE='缺少依赖'
ERROR_MESSAGE='编译失败'
# 输出结果
result_width=100
result_fix='='
result_prefix_num=10
result_postfix_num=15
# 输出结果顺序
result_sequence=("$SUCCESS" "$ERROR" "$MISSING" "$ALREADY")

#------------------------------------------------function define start------------------------------------------------
# 使用手册
function manual() {
  echo "Please ensure the directory ~/rpmbuild exists, And the script can execute in any directory."
  echo -e "\nUsage: bash spec.sh [OPTION]... [*.spec]... [OPTION]..."
  echo "  or:  bash spec.sh [*.spec]... [OPTION]... [*.spec]..."
  echo "rpmbuild all specfiles in the arguments, if it's empty, rpmbuild all specfiles in the SPECS directory"
  echo -e "\nArguments description"
  echo "  -f, --force         skip existence check, force rpmbuild"
  echo "  --thread=NUM        set the maximum number of executing processes"
  echo "  --result_fix=char   set the output result symbol"
  echo "  -* or --*           support all options in the rpmbuild"
}

# 命令行参数解析
function arr_spec_organize() {
  local ri=0
  local oi=0
  if [ $# -gt 0 ]; then
    for p in "$@"; do
      case $p in
      *.spec)
        arr[ri]=${p##*/}
        ((ri++))
        ;;
      -b[a-z])
        rpmbdf=$p
        ;;
      -f | --force)
        rpmbds=$p
        ;;
      --help)
        manual && exit
        ;;
      --thread=*)
        local _thread=${p##*=}
        if [[ $_thread =~ [0-9]+$ ]]; then
          while [[ $_thread = [0]* ]]; do
            _thread=${_thread#*0}
          done
          if [[ $_thread != '' && $_thread -gt 0 && $_thread -lt $thread_max_num ]]; then
            thread_max_num=$_thread
          fi
        fi
        ;;
      --result_fix=*)
        local _symbol="${p#*=}"
        [[ $_symbol =~ ^.$ ]] && result_fix=$_symbol
        ;;
      *)
        rpmbdo[oi]=$p
        ((oi++))
        ;;
      esac
    done
  fi
  # 命令行未传入spec文件则编译SPECS目录下所有spec文件
  if [ $ri -eq 0 ]; then
    for r in SPECS/*; do
      r=${r##*/}
      if [ "${r##*.}" = 'spec' ]; then
        arr[ri]=$r
        ((ri++))
      fi
    done
  fi
}

# 自动判定锁
function decide_lock() {
  local _lockfd=$global_lock_fd
  if [[ -n "$2" ]]; then
    _lockfd=$2
  fi
  case "$1" in
  "lock")
    read -ru "$_lockfd"
    ;;
  "unlock")
    echo >&"$_lockfd"
    ;;
  esac
}

# 加锁
function lock() {
  if $lock_switch; then
    decide_lock "lock" "$1"
  fi
}

# 解锁
function unlock() {
  if $lock_switch; then
    decide_lock "unlock" "$1"
  fi
}

# 已编译检查
function check_existence() {
  local name=''
  local version=''
  local _spec_detail=()
  while read -re line; do
    if [[ "$name" != '' && "$version" != '' ]]; then
      break
    fi
    IFS=':' read -ra _spec_detail <<<"$line"
    if [ ${#_spec_detail[@]} -eq 2 ]; then
      _spec_detail[0]=${_spec_detail[0]//[ 	]/''}
      _spec_detail[1]=${_spec_detail[1]//[ 	]/''}
      if [[ ${_spec_detail[0]} = [nN][aA][mM][eE] ]]; then
        name=${_spec_detail[1]}
      elif [[ ${_spec_detail[0]} = [vV][eE][rR][sS][iI][oO][nN] ]]; then
        version=${_spec_detail[1]}
      fi
    fi
  done <"SPECS/$1"
  # version是宏暂不处理，存在版本号校验问题使用-f触发强制编译即可
  if [[ $version != [0-9]* ]]; then
    version=''
  fi
  if [[ "$name" != '' ]]; then
    if [[ -n $(find ~/rpmbuild/RPMS -name "$name-$version*") || -n $(find ~/rpmbuild/RPMS -name "${name}[0-9]-$version*") ]]; then
      return 0
    fi
  fi
  return 1
}

# 编译执行
function exec_spec() {
  local src=$1
  # 查看是否已经编译完成 -> 源码编译不检查,强制编译不检查
  if [[ $rpmbdf != '-bs' && $rpmbds == '' ]] && check_existence "$src"; then
    return $ALREADY
  fi
  # 加载rpm包所需依赖 -> 源码编译不加载
  if [[ $rpmbdf != '-bs' ]]; then
    lock $builddep_lock_fd
    if ! dnf builddep "SPECS/$src"; then
      unlock $builddep_lock_fd
      return $MISSING
    fi
    unlock $builddep_lock_fd
  fi
  # 编译rpm二进制包
  if ! rpmbuild "$rpmbdf" "SPECS/$src" "${rpmbdo[@]}" "${custom_define_arr[@]}"; then
    return $ERROR
  fi
  return $SUCCESS
}

# 编译执行调用
function exec_compile() {
  exec_spec "$1"
  local exit_code=$?
  if $lock_switch; then
    # 退出码写入统计文件
    lock
    echo "$exit_code $1" >>$spec_count_file
    unlock
  else
    # 统计spec文件编译结果
    spec_compile_result "$exit_code $1"
  fi
}

# 编译rpm二进制包拷贝函数
function rpm_check_move() {
  if [ ! -e $success_dir ]; then
    mkdir -p $success_dir
  fi
  if [ ! -e $rpm_source ]; then
    mkdir -p $rpm_source
    createrepo_c $rpm_source --update
  fi
  # 开始拷贝二进制rpm包,将编译路径中的二进制rpm包移动到source目录下
  for rd in ~/rpmbuild/RPMS/*; do
    if [[ "$rd" = "$rpm_source" || $(find "$rd" -type f | wc -l) -eq 0 ]]; then
      continue
    fi
    if [[ "$1" = 'end' ]]; then
      cp -f "$rd"/* $success_dir
    fi
    mv -f "$rd"/* $rpm_source
  done
  # 检查createrepo_c包是否存在
  if [[ "$1" = 'start' ]] && ! which createrepo_c; then
    dnf install createrepo_c
    # 查询配置文件是否存在本地源
    while read -re line; do
      if [[ "$line" = "[local1]" ]]; then
        createrepo_c $rpm_source --update
        return 0
      fi
    done <$dnf_config
    echo -e "\n[local1] \nname=local1 \nbaseurl = file://$rpm_source \nenabled=1" >>$dnf_config
    createrepo_c $rpm_source --update
  fi
}

# 开启管道
function enable_fifo() {
  # 多进程管道文件绑定
  if [ -e "$lock_file" ]; then
    rm $lock_file
  fi
  mkfifo $lock_file
  exec 100<>$lock_file
  lock_fd=100
  rm $lock_file
  # 依赖加载管道文件绑定
  if [ -e "$builddep_lock_file" ]; then
    rm $builddep_lock_file
  fi
  mkfifo $builddep_lock_file
  exec 101<>$builddep_lock_file
  builddep_lock_fd=101
  rm $builddep_lock_file
  echo >&$builddep_lock_fd
  # 通用管道文件绑定
  if [ -e "$global_lock_file" ]; then
    rm $global_lock_file
  fi
  mkfifo $global_lock_file
  exec 102<>$global_lock_file
  global_lock_fd=102
  rm $global_lock_file
  echo >&$global_lock_fd
  # 创建令牌桶，限制并发数量
  if [ "$thread_max_num" -gt "$spec_count" ]; then
    thread_max_num=$spec_count
  fi
  for ((i = 0; i < "$thread_max_num"; i++)); do
    echo >&$lock_fd
  done
}

# 统计spec编译情况
function spec_compile_result() {
  local _spec_detail=()
  read -ra _spec_detail <<<"$1"
  case "${_spec_detail[0]}" in
  "$SUCCESS")
    success_spec_count_arr[success_spec_count]="${_spec_detail[1]}"
    ((success_spec_count++))
    ;;
  "$ALREADY")
    already_spec_count_arr[already_spec_count]="${_spec_detail[1]}"
    ((already_spec_count++))
    ;;
  "$MISSING")
    missing_spec_count_arr[missing_spec_count]="${_spec_detail[1]}"
    ((missing_spec_count++))
    ;;
  "$ERROR")
    error_spec_count_arr[error_spec_count]="${_spec_detail[1]}"
    ((error_spec_count++))
    ;;
  esac
}

# 统计类型输出结果
function finish_result_by_type() {
  for t in "$@"; do
    local _print_arr=()
    local _print_arr_count=0
    local _print_arr_tittle=''
    case "$t" in
    "$SUCCESS")
      _print_arr=("${success_spec_count_arr[@]}")
      _print_arr_count=$success_spec_count
      _print_arr_tittle="$SUCCESS_MESSAGE"
      ;;
    "$ALREADY")
      _print_arr=("${already_spec_count_arr[@]}")
      _print_arr_count=$already_spec_count
      _print_arr_tittle="$ALREADY_MESSAGE"
      ;;
    "$MISSING")
      _print_arr=("${missing_spec_count_arr[@]}")
      _print_arr_count=$missing_spec_count
      _print_arr_tittle="$MISSING_MESSAGE"
      ;;
    "$ERROR")
      _print_arr=("${error_spec_count_arr[@]}")
      _print_arr_count=$error_spec_count
      _print_arr_tittle="$ERROR_MESSAGE"
      ;;
    esac
    if [ $_print_arr_count -gt 0 ]; then
      # 打印
      echo -ne "$result_content_background\r"
      echo "$result_prev_fix $_print_arr_tittle "
      # 写日志
      if $log_switch; then
        echo "$_print_arr_tittle" >>$log_file
      fi
      if $log_history_switch; then
        echo "$_print_arr_tittle" >>$log_file_history
      fi
      for ((i = 0; i < _print_arr_count; i++)); do
        # 打印
        echo -ne "$result_content_background\r"
        echo "$result_prev_fix ${_print_arr[i]}"
        # 写日志
        if $log_switch; then
          echo "${_print_arr[i]}" >>$log_file
        fi
        if $log_history_switch; then
          echo "${_print_arr[i]}" >>$log_file_history
        fi
      done
      echo "$result_front_back"
    fi
  done
}

# 输出结果
function finish_result() {
  for ((i = 0; i < result_prefix_num; i++)); do
    result_prev_fix="$result_prev_fix$result_fix"
  done
  result_front_back=''
  result_content_background=''
  for ((i = 0; i < result_width; i++)); do
    result_front_back="$result_front_back$result_fix"
    if [ $i -lt $((result_width - result_postfix_num)) ]; then
      result_content_background="$result_content_background "
    else
      result_content_background="$result_content_background$result_fix"
    fi
  done
  # 统计个数输出
  local _result_count_print=()
  for c in "${result_sequence[@]}"; do
    case "$c" in
    "$SUCCESS")
      _result_count_print+=("$SUCCESS_MESSAGE: $success_spec_count")
      ;;
    "$ALREADY")
      _result_count_print+=("$ALREADY_MESSAGE $already_spec_count")
      ;;
    "$MISSING")
      _result_count_print+=(" $MISSING_MESSAGE $missing_spec_count")
      ;;
    "$ERROR")
      _result_count_print+=("$ERROR_MESSAGE $error_spec_count")
      ;;
    esac
  done
  for ((i = 0; i < ${#_result_count_print[@]} - 1; i++)); do
    _result_count_print[i]="${_result_count_print[i]} ,"
  done
  spec_count_print="spec文件总计: $spec_count , ${_result_count_print[*]}"
  # 执行时间
  runtime=$(($(date +%s) - script_start_time))
  runtime_print=''
  if [ "$runtime" -le 60 ]; then
    runtime_print="runtime $runtime 秒 "
  elif [ "$runtime" -le $((60 * 60)) ]; then
    runtime_print="runtime $((runtime / 60)) 分 $((runtime % 60)) 秒 "
  else
    runtime_print="runtime $((runtime / (60 * 60))) 时 $((runtime % (60 * 60) / 60)) 分 $((runtime % (60 * 60) % 60)) 秒 "
  fi
  # 输出结果
  echo -e "\n$result_front_back"
  echo -ne "$result_content_background\r"
  echo "$result_prev_fix $runtime_print"
  echo -ne "$result_content_background\r"
  echo "$result_prev_fix $spec_count_print"
  echo -e "$result_front_back"
  # 打印不同执行结果信息
  finish_result_by_type "${result_sequence[@]}"
}
#------------------------------------------------function define end-------------------------------------------------

#------------------------------------------------code execute start-------------------------------------------------
if ! cd ~/rpmbuild; then
  echo "cd ~/rpmbuild fail, Please ensure the directory ~/rpmbuild exists"
  exit
fi

# 命令行参数处理
arr_spec_organize "$@"
spec_count=${#arr[@]}
echo "==== spec文件总计: $spec_count ===="
[ "$spec_count" -gt 0 ] || exit
[ "$spec_count" -eq 1 ] || lock_switch=true

# rpm包路径检查和移动 -> 源码编译不检查
if [[ "$rpmbdf" != '-bs' ]]; then
  rpm_check_move 'start'
fi

# 刷新dnf缓存 -> 源码编译不刷新
if [[ "$rpmbdf" != '-bs' ]]; then
  echo '刷新dnf缓存中.......'
  dnf makecache
fi

# 开启管道
if $lock_switch; then
  enable_fifo
fi

# 清空统计记录
if $lock_switch; then
  echo -n >$spec_count_file
fi

# 日志写入日期头
if $log_switch; then
  date >$log_file
fi
if $log_history_switch; then
  echo >>$log_file_history && date >>$log_file_history
fi

# 多进程开始编译
for ((i = 0; i < spec_count; i++)); do
  if $lock_switch; then
    lock $lock_fd
    {
      exec_compile "${arr[i]}"
      unlock $lock_fd
    } &
  else
    exec_compile "${arr[i]}"
  fi
done
wait

# 统计spec文件编译结果
if $lock_switch; then
  while read -re line; do
    spec_compile_result "$line"
  done <$spec_count_file
  rm $spec_count_file
fi

# 如果有编译成功，拷贝并刷新 -> 源码编译不刷新
if [[ $success_spec_count -gt 0 && "$rpmbdf" != '-bs' ]]; then
  rpm_check_move 'end'
  createrepo_c $rpm_source --update
fi

# 记录日志并打印编译结果
finish_result
#------------------------------------------------code execute end--------------------------------------------------