kubernetes java app 部署使用harbor私服问题集合

1.kuboard/cluster 无法正确添加集群

bash 复制代码

 可以选择master 作为api server

2.文件列表 springboot-k8s-demo

bash 复制代码

jenkinsfile
dockerfile
k8s/deployment.yaml
k8s/service.yaml

jenkinsfile

bash 复制代码

pipeline {
    agent any  // 建议配置为已安装docker/kubectl的Jenkins节点
    environment {
        // 适配你的Harbor仓库地址
        HARBOR_URL     = '192.168.0.104:5000'
        HARBOR_PROJECT = 'javademo'
        IMAGE_NAME = "springboot-k8s-demo"
        IMAGE_TAG = "${BUILD_NUMBER}"  // 用构建号作为镜像标签
        K8S_NAMESPACE = "dev"          // K8s demo集群的命名空间
        FULL_IMAGE     = "${HARBOR_URL}/${HARBOR_PROJECT}/${IMAGE_NAME}:${IMAGE_TAG}"
        // Harbor登录凭证（建议用Jenkins凭证管理，避免明文）
        //HARBOR_USER = "admin"          // Harbor默认管理员账号
        //HARBOR_PASSWORD = "Harbor12345" // Harbor默认密码
    }

    stages {
        // 阶段1：拉取代码（替换为你的Git仓库地址）
        stage('Checkout Code') {
        //            steps {
        //                git url: 'http://192.168.0.254:8888/my-git-lab/springboot-k8s-demo.git', branch: 'dev'
        //            }
                    steps {
                        echo '=== 拉取代码 ==='
                        checkout scm
                        sh 'ls -la'
                        sh 'git log --oneline -3'
                    }
        }

        // 阶段2：构建并推送Docker镜像到Harbor
        stage('构建镜像') {
            steps {
                echo '=== 构建 Docker 镜像 ==='
                // 指定JDK 17环境（如果Jenkins节点有多个JDK版本）
                tool name: 'jdk17', type: 'hudson.model.JDK'
                // 指定Maven 3.9.6环境
                tool name: 'Maven-3.9.6', type: 'hudson.tasks.Maven$MavenInstallation'
                sh "docker build -t ${FULL_IMAGE} ."
                // 登录Harbor（注意：如果Harbor是自签名证书，需先配置docker信任）
//                echo '=== 推送镜像到 Harbor ==='
//                sh "docker login 192.168.0.104:5000 -u ${HARBOR_USER} -p ${HARBOR_PASSWORD}"
//                sh "docker push ${IMAGE_NAME}:${IMAGE_TAG}"
 //               sh "docker logout 192.168.0.104:5000"
            }
        }

                stage('推送到 Harbor') {
                    steps {
                        echo '=== 推送镜像到 Harbor ==='
                        withCredentials([usernamePassword(
                            credentialsId: 'harbor-credentials',
                            usernameVariable: 'HARBOR_USER',
                            passwordVariable: 'HARBOR_PASSWORD')]) {
                            sh '''
                                docker login ${HARBOR_URL} \
                                  -u ${HARBOR_USER} -p ${HARBOR_PASSWORD}
                                docker push  ${FULL_IMAGE}
                                echo "✅ 镜像推送成功：${FULL_IMAGE}"
                            '''
                        }
                    }
                }

//         stage('推送到 Harbor') {
//             steps {
//                 echo '=== 推送镜像到 Harbor ==='
//                 sh "docker login 127.0.0.1:5000 -u ${HARBOR_USER} -p ${HARBOR_PASSWORD}"
//                 sh "docker push ${IMAGE_NAME}:${IMAGE_TAG}"
//                 sh "docker logout 127.0.0.1:5000"
//             }
//         }

        // 阶段3：部署到K8s demo集群
        stage('部署到 K8s') {
            steps {
                echo '=== 部署到 K8s ==='
                // 1. 关键：先复制Deployment模板，再替换（避免原文件缓存问题）
                sh "cp k8s/deployment.yaml k8s/deployment-temp.yaml"
                // 2，替换K8s部署清单中的镜像地址 用双引号包裹sed命令，确保变量解析（之前是单引号导致变量未替换）
                sh "sed -i \"s|IMAGE_PLACEHOLDER|${FULL_IMAGE}|g\" k8s/deployment-temp.yaml"
                // 3. 打印替换结果（必须确认是实际数字，而非${BUILD_NUMBER}）
                sh "echo '=== 替换后的镜像地址 ==='"
                sh "grep 'image:' k8s/deployment-temp.yaml"
                // 4. 创建命名空间（如果不存在）
                sh "kubectl create namespace ${K8S_NAMESPACE} --dry-run=client -o yaml | kubectl apply -f -"
                // 5. 部署临时文件（而非原文件）
                sh "kubectl apply -f k8s/deployment-temp.yaml -n ${K8S_NAMESPACE}"
                sh "kubectl apply -f k8s/service.yaml -n ${K8S_NAMESPACE}"
                // 验证部署状态
                //sh "kubectl rollout status deployment/springboot-k8s-demo -n ${K8S_NAMESPACE}"
                // 验证部署状态：延长超时到5分钟，失败时输出Pod日志（核心！）
                sh """
                    set -e
                    # 延长rollout超时到5分钟，和Deployment的progressDeadlineSeconds一致
                    kubectl rollout status deployment/springboot-k8s-demo -n ${K8S_NAMESPACE} --timeout=5m || (
                        echo '========================================='
                        echo '❌ 部署超时，开始排查Pod错误：'
                        echo '-----------------------------------------'
                        echo '1. Pod状态：'
                        kubectl get pods -n ${K8S_NAMESPACE}
                        echo '-----------------------------------------'
                        echo '2. Pod详细错误（Events）：'
                        kubectl describe pods -n ${K8S_NAMESPACE} | grep -A 40 "Events"
                        echo '-----------------------------------------'
                        echo '3. 容器启动日志：'
                        kubectl logs -l app=springboot-k8s-demo -n ${K8S_NAMESPACE} --tail=100
                        echo '========================================='
                        exit 1
                    )
                """
            }
        }
    }

    post {
        // 构建失败通知
        failure {
            echo "部署到K8s demo集群失败！构建号：${BUILD_NUMBER}"
        }
        // 清理本地镜像
        always {
            sh "docker rmi ${IMAGE_NAME}:${IMAGE_TAG} || true"
        }
    }
}

dockerfile

bash 复制代码

# 阶段1：构建 Jar 包
FROM 192.168.0.104:5000/javademo/openjdk:17-jdk-slim AS builder
WORKDIR /app

RUN apt update && apt install -y --no-install-recommends wget tar && \
    wget https://archive.apache.org/dist/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz -P /tmp && \
    tar -zxvf /tmp/apache-maven-3.9.6-bin.tar.gz -C /usr/local && \
    ln -s /usr/local/apache-maven-3.9.6/bin/mvn /usr/bin/mvn && \
    rm -rf /tmp/apache-maven-3.9.6-bin.tar.gz && \
    apt clean && rm -rf /var/lib/apt/lists/*

RUN mkdir -p /root/.m2 && \
    echo "<settings><mirrors><mirror><id>aliyunmaven</id><name>阿里云中央仓库</name><url>https://maven.aliyun.com/repository/public</url><mirrorOf>central</mirrorOf></mirror></mirrors></settings>" \
    > /root/.m2/settings.xml

COPY pom.xml .
RUN mvn dependency:go-offline -B

COPY src ./src
RUN mvn package -DskipTests

# 阶段2：运行 Jar 包
FROM 192.168.0.104:5000/javademo/openjdk:17-jdk-slim
WORKDIR /app

# 直接用 root 运行，由 k8s securityContext 控制权限
COPY --from=builder /app/target/*.jar app.jar

ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0"

EXPOSE 8080

# 直接调用 java，不通过 sh
ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-XX:MaxRAMPercentage=75.0", "-jar", "app.jar"]

k8s/deployment.yaml

bash 复制代码

apiVersion: apps/v1
kind: Deployment
metadata:
  name: springboot-k8s-demo
  labels:
    app: springboot-k8s-demo
spec:
  replicas: 2
  progressDeadlineSeconds: 300
  selector:
    matchLabels:
      app: springboot-k8s-demo
  template:
    metadata:
      labels:
        app: springboot-k8s-demo
    spec:
      imagePullSecrets:
        - name: harbor-credentials
      containers:
        - name: springboot-k8s-demo
          image: IMAGE_PLACEHOLDER
          imagePullPolicy: Always
          ports:
            - containerPort: 8080
          securityContext:
            runAsUser: 0
            runAsGroup: 0
            appArmorProfile:
              type: Unconfined
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: 500m
              memory: 512Mi
          livenessProbe:
            httpGet:
              path: /actuator/health
              port: 8080
            initialDelaySeconds: 60
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /actuator/health
              port: 8080
            initialDelaySeconds: 40
            periodSeconds: 5
            timeoutSeconds: 3
            failureThreshold: 3
          env:
            - name: SPRING_PROFILES_ACTIVE
              value: "default"

k8s/service.yaml

bash 复制代码

apiVersion: v1
kind: Service
metadata:
  name: springboot-k8s-demo
spec:
  type: NodePort  # demo集群测试用NodePort
  selector:
    app: springboot-k8s-demo
  ports:
    - port: 8080
      targetPort: 8080
      nodePort: 30080  # 节点端口（范围30000-32767）

3.打出的jar包缺少可执行类

bash 复制代码

czh@k8s-master-03:~$ 
sudo crictl logs \
   $(sudo crictl ps -a | grep springboot | head -1 | awk '{print $1}') 2>&1
no main manifest attribute, in app.jar 
czh@k8s-master-01:~$ 
sudo crictl logs \
   $(sudo crictl ps -a | grep springboot | head -1 | awk '{print $1}') 2>&1
no main manifest attribute, in app.jar

没有 Main-Class，导致 no main manifest attribute。

原因： springboot 2.6.3 

    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-dependencies</artifactId>
                <version>${spring-boot.version}</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>


 <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <version>${spring-boot.version}</version>
                <configuration>
                    <!-- 确保主类路径正确（替换为你实际的启动类全路径） -->
                    <mainClass>com.demo.SpringbootK8sDemoApplication</mainClass>
                    <!-- 关键：删除 skip=true，或设置为 false，让插件正常工作 -->
                    <skip>false</skip>
                </configuration>
                <executions>
                    <execution>
                        <goals>
                            <goal>repackage</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

4.AppArmor 无权限执行(Exit Code: 127 --- 命令找不到，sh 不存在或不可执行）

bash 复制代码

修复 Exit Code 127（sh 找不到）
Exit Code 127 + libc.so.6 Permission denied 说明基础镜像 openjdk:17-jdk-slim 是 debian slim，sh 被 AppArmor 限制。


最新日志：
master-01  master-03 (实际部署的节点)
sudo crictl logs \
  $(sudo crictl ps -a | grep springboot | head -1 | awk '{print $1}') 2>&1


彻底禁用所有节点 AppArmor(mster worker)
    sudo systemctl stop apparmor 2>/dev/null
    sudo systemctl disable apparmor 2>/dev/null
    sudo aa-teardown 2>/dev/null
# 重启 containerd
sudo systemctl restart containerd

Dockerfile
直接用 java，不用 sh -c
ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-XX:MaxRAMPercentage=75.0", "-jar", "app.jar"]

# 删除旧的失败 pod
kubectl delete pods -n dev --all --force --grace-period=0


或者修改 Dockerfile 的 ENTRYPOINT(验证无效)

dockerfile

bash 复制代码

# 阶段2：运行 Jar 包
FROM 192.168.0.104:5000/javademo/openjdk:17-jdk-slim
WORKDIR /app

RUN groupadd -r dev && useradd -r -g dev dev

COPY --from=builder /app/target/*.jar app.jar

RUN chown -R dev:dev /app

# 确保 sh 可执行
RUN chmod 755 /bin/sh

USER dev

ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0"

EXPOSE 8080

# 直接用 java 命令，不通过 sh
ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-XX:MaxRAMPercentage=75.0", "-jar", "app.jar"]

5.dev无权限执行问题: FailedToRetrieveImagePullSecret harbor-credentials --- dev 命名空间缺少 Harbor认证

bash 复制代码

第一步：直接查看容器崩溃日志
# 用 crictl 查看
sudo crictl ps -a | grep springboot
sudo crictl logs $(sudo crictl ps -a | grep springboot | head -1 | awk '{print $1}')

第二步：确认 harbor-credentials secret 是否存在	
kubectl get secret harbor-credentials -n dev
如果不存在：

kubectl create secret docker-registry harbor-credentials \
  --docker-server=192.168.0.104:5000 \
  --docker-username=admin \
  --docker-password=123456\
  -n dev	

第三步：需要在 k8s/deployment.yaml 中添加：
spec:
  template:
    spec:
      imagePullSecrets:
      - name: harbor-credentials  # ← 添加这个

6.成功部署

部署的名称空间

部署的app

7.总结：

很多都是权限及国内镜像不通问题，部署的各种异常

kubernetes java app 部署使用harbor私服 问题集合

1.kuboard/cluster 无法正确添加集群

2.文件列表 springboot-k8s-demo

jenkinsfile

dockerfile

k8s/deployment.yaml

k8s/service.yaml

3.打出的jar包缺少可执行类

4.AppArmor 无权限执行(Exit Code: 127 --- 命令找不到，sh 不存在或不可执行）

dockerfile

5.dev无权限执行问题: FailedToRetrieveImagePullSecret harbor-credentials --- dev 命名空间缺少 Harbor认证

6.成功部署

部署的名称空间

部署的app

7.总结：

kubernetes java app 部署使用harbor私服问题集合