server环境：Centos 8
Gitlab版本：Gitlab v14.9
安装方式：docker
下面是具体的操作及文档
注意点：
- 其中JAVA为扫描增量代码，即老代码不会去扫描，只扫描提交上来后更改行的代码，因为如果该文件中有其他历史代码，修改工作量会很大，更适用于老项目的维护开发。
- Python为全量扫描，所以会扫描文件中的历史代码，适用于新项目的开发。
那么两种扫描方式我都已经给出了解决方案，如果需要python变成增量扫描，可以自己根据JAVA的逻辑去实现，当然肯定需要大量的调试，JAVA的这块我就调试了很久才出来的结果。
#参考材料
p3c代码仓库：
https://github.com/alibaba/p3c 下面脚本中的一个p3c-pmd-2.1.1-jar-with-dependencies.jar包，就是通过下载仓库代码，mvn package而来
https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_server_side_hooks
https://blog.csdn.net/gdyycn/article/details/123314790
https://github.com/geeeeeeeeek/git-recipes/wiki/5.4-Git-%E9%92%A9%E5%AD%90%EF%BC%9A%E8%87%AA%E5%AE%9A%E4%B9%89%E4%BD%A0%E7%9A%84%E5%B7%A5%E4%BD%9C%E6%B5%81
https://www.cnblogs.com/liruixin/p/15893896.html
https://zhuanlan.zhihu.com/p/548640101?utm_id=0
#参考文章：
https://www.cnblogs.com/c-moon/p/16825766.html
https://blog.csdn.net/xiaoll880214/article/details/105916898
https://github.com/github/platform-samples/tree/master/pre-receive-hooks
==========================部署操作如下，需要docker exec 进入gitlab容器后进行操作=======================================
vi /etc/gitlab/gitlab.rb
#添加一行如下
gitlab_shell['custom_hooks_dir'] = "/opt/gitlab/embedded/service/gitlab-shell/hooks"
重启gitlab以应用上面修改的配置
gitlab-ctl restart
mkdir -p /var/opt/gitlab/gitlab-hooks/
touch /var/opt/gitlab/gitlab-hooks/java-check-list.txt
touch /var/opt/gitlab/gitlab-hooks/python-check-list.txt
touch /var/opt/gitlab/gitlab-hooks/csharp-check-list.txt
mkdir -p /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d
chmod 755 /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d
chown -R git:git /opt/gitlab/embedded/service/gitlab-shell/hooks
vi /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d/pre-receive.sh
chmod +x /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d/pre-receive.sh
把文件复制到gitlab镜像的/opt/目录下
cd /opt/
chmod +x p3c-pmd-2.1.1-jar-with-dependencies.jar
官方下载openjdk，这个压缩包openlogic-openjdk-11.0.23+9-linux-x64.tar.gz,放到gitlab镜像的/opt/目录下
cd /opt/
解压到docker gitlab容器的目录下，我这里解压到/opt/jdk-11.0.23目录下，这个目录跟下面脚本中的JAVA_HOME内容保持一致
#解压openjdk压缩包
tar -xvf openlogic-openjdk-11.0.23+9-linux-x64.tar.gz
#重命名一下解压出来的文件夹为 jdk-11.0.23
mv openlogic-openjdk-11.0.23+9-linux-x64 jdk-11.0.23
#安装python所需要的检查工具
pip3 install pycodestyle --proxy=http://10.161.238.16:3128
pip3 install pep8 --proxy=http://10.161.238.16:3128
#安装eslint
export http_proxy=http://10.161.238.16:3128 && export https_proxy=http://10.161.238.16:3128
mkdir -p /opt/nodejs
cd /opt/nodejs
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install nodejs -y
#最新地址 淘宝 NPM 镜像站喊你切换新域名啦!
npm config set registry https://registry.npmmirror.com
npm install -g eslint @antfu/eslint-config
export http_proxy=http://10.161.238.16:3128 && export https_proxy=http://10.161.238.16:3128
#安装awk
apt-get install gawk
这里gitlab容器内有两个目录比较常用：
/opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d
/var/opt/gitlab/gitlab-hooks
pre-receive文件脚本内容如下：
#!/bin/bash
####### 初始化变量部分 #########
echo -e '\033[33m==================Git Hooks version 1.0, powered By Digi M&O Team, if you have any questions, please contact us.==================\033[0m'
## 定义java_home变量 需要修改你配置的java_home
JAVA_HOME=/opt/jdk-11.0.23
## 定义p3c-pmd工具jar包位置
P3C_PMD_TOOL_HOME=/opt/p3c-pmd-2.1.1-jar-with-dependencies.jar
## 是否开启commit message的校验：0是，1否
CHECK_COMMIT_MESSAGE_ON=0
## 是否开启代码检查：0是，1否
CHECK_CODE_RULE_ON=0
## 是否校验master上的pom文件是否包含snapshot：0是，1否
CHECK_MASTER_POM_SNAPSHOT_ON=1
## 注释内容最小长度，默认20
COMMIT_MESSAGE_MIN_LENGTH=20
### 代码校验规则：0使用阿里云P3C规则，1使用checkStyle
JAVA_CODE_RULE_TYPE=0
### 代码校验规则：0使用pycodestyle规则，1使用pep8
PYTHON_CODE_RULE_TYPE=0
## 定义提交开头类型字符规则
## e.g: fix:测试提交bug修复，Bug编号#12
TYPE_LIST=(
'feat:' #新功能feature
'update:' #在feat内修改
'fix:' #修补bug
'docs:' #文档
'style:' #格式化，不影响代码运行的变动
'refactor:' #重构
'pref:' #性能优化
'test:' #增加测试
'chore:' #构建过程或辅助工具的变动
'Merge'
'merge'
#'[ci skip]' #忽略校验
)
# Java项目CheckList
JAVA_CHECK_LIST='/var/opt/gitlab/gitlab-hooks/java-check-list.txt'
PYTHON_CHECK_LIST='/var/opt/gitlab/gitlab-hooks/python-check-list.txt'
CSHARP_CHECK_LIST='/var/opt/gitlab/gitlab-hooks/csharp-check-list.txt'
# 判断项目是否在java检查名单内
IS_GIT_JAVA_CHECK_LIST=`/usr/bin/grep -nw "${GL_REPOSITORY}" $JAVA_CHECK_LIST`
# 判断项目是否在python检查名单内
IS_GIT_PYTHON_CHECK_LIST=`/usr/bin/grep -nw "${GL_REPOSITORY}" $PYTHON_CHECK_LIST`
# 判断项目是否在csharp检查名单内
IS_GIT_CSHARP_CHECK_LIST=`/usr/bin/grep -nw "${GL_REPOSITORY}" $CSHARP_CHECK_LIST`
# 声明一个关联数组用于保存文件名和行号的映射
declare -A file_lines_map
# 创建一个临时文件存放文件名和行号的映射
#temp_file=$(mktemp)
temp_file=""
#echo "tempfile:" $temp_file
# pmd输出内容临时文件
#pmd_out_temp_file=$(mktemp)
pmd_out_temp_file=""
#echo "pmd_out_temp_file:" $pmd_out_temp_file
## 获取当前路径
BASE_PATH=$(cd `dirname $0`; pwd)
#echo 'BASE_PATH: '$BASE_PATH
#定义和组装校验规则
declare -a regex_list
arrLen=${#TYPE_LIST[@]}
for ((i=0;i<$arrLen;i++)) do
regex_list[i]='^'${TYPE_LIST[i]}
done
regex_list[$arrLen+1]='^[ci skip]:'
#echo "reg_list=== "${regex_list[@]}
separator="|"
## 合并成一个完整的正则表达式
regex="$( printf "${separator}%s" "${regex_list[@]}" )"
#echo "type regex: "$regex
## 去除头部的 |
regex=${regex:${#separator}}
#echo "regex: "$regex
## 定义注释出错提示信息
tips_msg="$( printf "${separator}%s" "${TYPE_LIST[@]}" )"
tips_msg=${tips_msg:${#separator}}
validate_code_ignore=0
####### 初始化变量部分 #########
####### 校验部分：注释校验&代码分析###########
## 校验commit message
validate_commit_message()
{
oldrev=$(git rev-parse $1)
newrev=$(git rev-parse $2)
refname="$3"
#echo 'Old version: '$oldrev
#echo 'New version: '$newrev
#echo 'Branch: '$refname
## git 命令
#GITCMD="git"
## 按时间倒序列出 commit 找出两个版本之间差异的版本号集合oldrev~newrev
commitList=`git rev-list $oldrev..$newrev`
#echo 'commitList: '$commitList
split=($commitList)
#echo 'split: '$split
# 遍历数组
for s in ${split[@]}
do
#echo "$s"
#通过版本号获取仓库中对象实体的类型、大小和内容的信息
#比如提交人、作者、邮件、提交时间、提交内容等
currentContent=`git cat-file commit $s`
#echo 'Commit obj: '$currentContent
#获取提交内容
msg=`git cat-file commit $s | sed '1,/^$/d'`
#echo 'msg: '$msg
if [[ "$variable" == *"--ignore"* ]]; then
validate_code_ignore=1
fi
## merge合并分之直接放行
if [[ $msg == *"Merge branch"* ]]; then
echo "Merge branch...skip the checking"
else
## 做内容校验
match=`echo $msg | grep -nE "(${regex})"`
#echo 'Match result: '$match
## 找到匹配说明是符合规范的
if [ "${match}" != "" ]; then
## 校验注释长度
#msg_length=${#msg}
#echo "Msg length: ${msg_length}"
#if [[ ${msg_length} -lt ${COMMIT_MESSAGE_MIN_LENGTH} ]]; then
# echo -e "Error: Commit message should be bigger than ${COMMIT_MESSAGE_MIN_LENGTH} and current commit message length: ${msg_length}"
# exit 1
#fi
### 找到匹配内容做相应处理，如fix, 校验pom文件等
#if [[ "${match}" =~ "fix:" ]]; then
## 如果是修补bug，规范有点获取到fix中的ID，然后调用禅道对外的API关闭，其他场景类似
#fi
# 是否开启校验和master分之
# isMaster=$(echo $refname | grep "master$")
# if [ $CHECK_MASTER_POM_SNAPSHOT_ON == 0 ] && [ -n "$isMaster" ]; then
# # 如果是master分之，并且pom文件发生了变更，判断pom文件是否含有sonapshot的引用
# pomfile=`git diff --name-only ${oldrev} ${newrev} | grep -e "pom\.xml"`
# if [[ "${pomfile}" != "" ]]; then
# #echo $pomfile
# ## 获取pom文件更新的内容
# pomcontent=`git show $newrev:$pomfile`
# #echo $pomcontent
# ## 校验pom文件是否包含snapshot版本
# if [[ $pomcontent =~ 'SNAPSHOT' ]]; then
# echo -e "Error: Snapshot version cannot exist in master branch!"
# exit 1
# fi
# fi
# fi
## 其他操作
echo "Commit Success!"
else
echo -e "Error: Commit comments message should be started with [${tips_msg}]..."
exit 1
fi
fi
done
}
## 代码校验
validate_java_code_rules()
{
echo 'Start java code analysis!'
oldrev=$(git rev-parse $1)
newrev=$(git rev-parse $2)
refname="$3"
#echo 'Old version: '$oldrev
#echo 'New version: '$newrev
#echo 'Branch: '$refname
FILES=`git diff --name-only ${oldrev} ${newrev} | grep -e "\.java$"`
if [ -n "$FILES" ]; then
# 初始化临时文件
temp_file=$(mktemp)
pmd_out_temp_file=$(mktemp)
#${GL_REPOSITORY}是gitlab的内置变量，格式为project-项目id，即project-21
TEMPDIR=$BASE_PATH/"tmp/java_tmp"/${GL_REPOSITORY}
for FILE in ${FILES}; do
mkdir -p "${TEMPDIR}/`dirname ${FILE}`" >/dev/null
git show $newrev:$FILE > ${TEMPDIR}/${FILE}
# 获取每个文件的变更行号并添加到临时文件中
# 检查文件是否在旧版本中存在
if git ls-tree -r $oldrev --name-only | grep -q "^${FILE}$"; then
# 处理已有文件的变更行号
git diff --unified=0 "$oldrev:$FILE" "$newrev:$FILE" | grep -E "^@@ " | awk -v fname="${TEMPDIR}/${FILE}" '
{
split($3, arr, /[,+]/)
start_line = arr[2]
num_lines = (length(arr) > 2) ? arr[3] : 1
for (i = start_line; i < start_line + num_lines; i++) {
print fname ":" i
}
}' >> "$temp_file"
else
# 处理新增文件的所有行号
awk -v fname="${TEMPDIR}/${FILE}" '{print fname ":" NR}' "${TEMPDIR}/${FILE}" >> "$temp_file"
fi
done;
#MAIN_FILE_PATH=$TEMPDIR'/src/main'
MAIN_FILE_PATH=$TEMPDIR
#echo 'Temp update files path: '$MAIN_FILE_PATH
# 读取临时文件并更新 file_lines_map 数组
while IFS=: read -r fname line_num; do
#echo "File: $fname, Line: $line_num"
if [ -z "${file_lines_map[$fname]}" ]; then
file_lines_map[$fname]="$line_num"
else
file_lines_map[$fname]="${file_lines_map[$fname]} $line_num"
fi
done < "$temp_file"
# 删除临时文件
rm -f "$temp_file"
#FILES_TO_CHECK=`find $MAIN_FILE_PATH -name '*.java'`
#echo 'Check files:'${FILES_TO_CHECK}
echo 'Aliyun p3c-pmd check starting.....'
#echo 'Current shell Path:' $BASE_PATH
#echo 'JAVA_HOME:' $JAVA_HOME
#echo 'Root directory for java sources: '$MAIN_FILE_PATH
#RESULT=""
if [[ $JAVA_CODE_RULE_TYPE == 0 ]]; then
## 需要把阿里云P3C的插件包p3c-pmd-2.1.1-jar-with-dependencies.jar放在/opt/目录下，当然你也可以自定义。jar包来源为通过下载https://github.com/alibaba/p3c仓库代码，mvn package p3c-pmd模块代码而来
echo -e '\033[31m==================Code analysis for Aliyun-p3c. The following output are the problem codes that need to be fixed==================\033[0m'
$JAVA_HOME/bin/java -Dpmd.language=en -cp $P3C_PMD_TOOL_HOME net.sourceforge.pmd.PMD -d $TEMPDIR -R rulesets/java/ali-comment.xml,rulesets/java/ali-concurrent.xml,rulesets/java/ali-constant.xml,rulesets/java/ali-exception.xml,rulesets/java/ali-flowcontrol.xml,rulesets/java/ali-naming.xml,rulesets/java/ali-oop.xml,rulesets/java/ali-other.xml,rulesets/java/ali-set.xml -f text > "$pmd_out_temp_file"
# 将关联数组转换为适合 AWK 使用的格式（键和值拼接成字符串）
awk_input=""
for key in "${!file_lines_map[@]}"; do
value="${file_lines_map[$key]}"
awk_input+="$key:$value;"
done
output=$(awk -F: -v pmd_out_temp_file="$pmd_out_temp_file" -v awk_input="$awk_input" '
BEGIN {
# 分号分隔每个
split(awk_input, pairs, ";")
for (i in pairs) {
if (pairs[i] != "") {
split(pairs[i], kv, ":")
key = kv[1]
value = kv[2]
file_lines_map[key] = value
}
}
match_line = 0
# 在这里可以使用 file_lines_map 数组
#for (key in file_lines_map) {
# print "key:", key, "value:", file_lines_map[key]
#}
}
{
file1=gensub(/[[:space:]]/, "", "g", $1)
#print "file1: " file1
line_num=$2
#print "line_num: " line_num
message=$3
#print "message: " message
modified_path=gensub(/.*\\src/, "src", "g", file1)
gsub(/\\/, "/", modified_path) # 将 modified_path 中的 \ 替换为 /
found = 0
for (key in file_lines_map) {
#print "key: " key
#print "modified_path: " modified_path
if (key == modified_path) {
print "found: " found
found = 1
break
}
}
if (found) {
changed_lines=file_lines_map[modified_path]
if (index(" " changed_lines " ", " " line_num " ")) {
RESULT=RESULT file1 ":" line_num ":" message "\\n"
match_line++
}
} else {
print "modified_path not found in file_lines_map"
}
}
END {
#if (found) {
# printf "%s", RESULT
# system("rm -rf " ENVIRON["TEMPDIR"]);
# exit(1);
# }
#printf "%s", RESULT
#print "SCAN_RESULT=" RESULT
print "MATCH_COUNT=" match_line
printf "SCAN_RESULT=\"%s\"\n", RESULT
}' "$pmd_out_temp_file"
)
# 删除临时文件
rm -f "$pmd_out_temp_file"
#SCAN_RESULT=$(echo "$output" | grep '^SCAN_RESULT=' | sed 's/^SCAN_RESULT=//')
#SCAN_RESULT=$(echo "$output" | sed -n '/^SCAN_RESULT=/,$p')
SCAN_RESULT=$(echo "$output" | awk '/^SCAN_RESULT=/{flag=1} flag')
processed_result=processed_result=$(echo "$SCAN_RESULT" | sed 's/^SCAN_RESULT="//; s/\\n"$//')
SCAN_RESULT=$processed_result
#SCAN_RESULT=$(echo "$output" | sed -n '/^SCAN_RESULT=/,$p')
#SCAN_RESULT=$(echo "$output" | awk '/^SCAN_RESULT=/{flag=1} flag' | printf "%s\n" "$(cat)")
MATCH_COUNT=$(echo "$output" | grep '^MATCH_COUNT=' | sed 's/^MATCH_COUNT=//')
#RESULT=$?
#echo $MATCH_COUNT
if [ "$MATCH_COUNT" -gt 0 ]; then
echo -e $SCAN_RESULT
rm -rf $TEMPDIR
exit 1;
fi
elif [[ $JAVA_CODE_RULE_TYPE == 1 ]]; then
## 需要CheckStyle插件包checkstyle-8.16-all与该脚本在同级目录下，并且需要对应的CheckStyle.xml模板文件e.g:Cheetah_Checkstyle_ruleset.xml
echo -e '\033[31m==================Code analysis for CheckStyle. The following output are the problem codes that need to be fixed==================\033[0m'
CHECK_RESULT=`$JAVA_HOME/bin/java -jar $BASE_PATH/checkstyle-8.16-all.jar -c $BASE_PATH/Cheetah_Checkstyle_ruleset.xml $MAIN_FILE_PATH`
if [[ $CHECK_RESULT =~ "[WARN]" ]]; then
echo $CHECK_RESULT | sed 's/\[WARN\]/\n/g'
rm -rf $TEMPDIR
exit 1
fi
else
## 不支持的检查操作
echo "Unsupported code validation rule,Please contact the administrator to check the configuration of [JAVA_CODE_RULE_TYPE] in pre-receive script!"
rm -rf $TEMPDIR
exit 1
fi
echo 'End code analysis!'
rm -f "$temp_file"
rm -f "$pmd_out_temp_file"
rm -rf $TEMPDIR
fi
}
validate_python_code_rules()
{
#echo 'Start python code analysis!'
#echo 'Skip python code analysis!, has no python rulesets'
echo 'Start python code analysis!'
oldrev=$(git rev-parse $1)
newrev=$(git rev-parse $2)
refname="$3"
#echo 'Old version: '$oldrev
#echo 'New version: '$newrev
#echo 'Branch: '$refname
FILES=`git diff --name-only ${oldrev} ${newrev} | grep -e "\.py$"`
if [ -n "$FILES" ]; then
#${GL_REPOSITORY}是gitlab的内置变量，格式为project-项目id，即project-21
TEMPDIR=$BASE_PATH/"tmp/python_tmp"/${GL_REPOSITORY}
for FILE in ${FILES}; do
mkdir -p "${TEMPDIR}/`dirname ${FILE}`" >/dev/null
git show $newrev:$FILE > ${TEMPDIR}/${FILE}
done;
#MAIN_FILE_PATH=$TEMPDIR'/src/main'
MAIN_FILE_PATH=$TEMPDIR
#echo 'Temp update files path: '$MAIN_FILE_PATH
#FILES_TO_CHECK=`find $MAIN_FILE_PATH -name '*.py'`
#echo 'Check files:'${FILES_TO_CHECK}
echo 'Pycodestyle check starting.....'
#echo 'Current shell Path:' $BASE_PATH
#echo 'Root directory for python sources: '$MAIN_FILE_PATH
if [[ $PYTHON_CODE_RULE_TYPE == 0 ]]; then
## 需要提前安装工具, pip3 install pycodestyle
echo -e '\033[31m==================Code analysis for Pycodestyle. The following output are the problem codes that need to be fixed==================\033[0m'
pycodestyle $TEMPDIR
RESULT=$?
#echo $RESULT
if [ $RESULT -gt 0 ]; then
rm -rf $TEMPDIR
exit 1;
fi
elif [[ $PYTHON_CODE_RULE_TYPE == 1 ]]; then
## 需要提前安装工具, pip3 install pep8
echo -e '\033[31m==================Code analysis for CheckStyle. The following output are the problem codes that need to be fixed==================\033[0m'
CHECK_RESULT=`pep8 --show-source --show-pep8 $TEMPDIR`
if [[ $CHECK_RESULT =~ "[WARN]" ]]; then
echo $CHECK_RESULT | sed 's/\[WARN\]/\n/g'
rm -rf $TEMPDIR
exit 1
fi
else
## 不支持的检查操作
echo "Unsupported code validation rule,Please contact the administrator to check the configuration of [PYTHON_CODE_RULE_TYPE] in pre-receive script!"
rm -rf $TEMPDIR
exit 1
fi
echo 'End code analysis!'
rm -rf $TEMPDIR
fi
}
validate_csharp_code_rules()
{
#echo 'Start csharp code analysis!'
echo 'Skip csharp code analysis!, has no csharp rulesets'
}
####### 校验部分：注释校验&代码分析###########
####### 执行入口###########
pre_receive()
{
#代码规则检查
if [[ $CHECK_CODE_RULE_ON == 0 ]]; then
if [ -n "$IS_GIT_JAVA_CHECK_LIST" ]; then
#commit message 校验
if [[ $CHECK_COMMIT_MESSAGE_ON == 0 ]]; then
validate_commit_message $1 $2 $3
fi
if [[ $validate_code_ignore == 0 ]]; then
validate_java_code_rules $1 $2 $3
fi
elif [ -n "$IS_GIT_PYTHON_CHECK_LIST" ]; then
#commit message 校验
if [[ $CHECK_COMMIT_MESSAGE_ON == 0 ]]; then
validate_commit_message $1 $2 $3
fi
if [[ $validate_code_ignore == 0 ]]; then
validate_python_code_rules $1 $2 $3
fi
elif [ -n "$IS_GIT_CSHARP_CHECK_LIST" ]; then
#commit message 校验
if [[ $CHECK_COMMIT_MESSAGE_ON == 0 ]]; then
validate_commit_message $1 $2 $3
fi
if [[ $validate_code_ignore == 0 ]]; then
validate_csharp_code_rules $1 $2 $3
fi
fi
fi
}
# update hook触发会带参数执行if逻辑
# hooks脚本触发无参数执行else逻辑
if [ -n "$1" -a -n "$2" -a -n "$3" ]; then
# Output to the terminal in command line mode - if someone wanted to
# resend an email; they could redirect the output to sendmail
# themselves
pre_receive $2 $3 $1
#echo $1'+'$2'+'$3
else
while read oldrev newrev refname
do
pre_receive $oldrev $newrev $refname
#echo $oldrev' '$newrev' '$refname
done
fi
####### 执行入口###########
exit 0