今天在k8s上腾了nacos服务,费了不少周折,但最终成功了,分享一下。
首先是要安装存储类
用于nacos存储数据和日志。我是把ceph存储持载到本地使用,比较简单,但没有直接用csi-rbd灵活。主要是我司ceph比较老,不支持k8s。
yml
apiVersion: v1
kind: Namespace
metadata:
name: local-path-storage
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: local-path-provisioner-service-account
namespace: local-path-storage
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: local-path-provisioner-role
namespace: local-path-storage
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: local-path-provisioner-role
rules:
- apiGroups: [""]
resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: local-path-provisioner-bind
namespace: local-path-storage
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: local-path-provisioner-role
subjects:
- kind: ServiceAccount
name: local-path-provisioner-service-account
namespace: local-path-storage
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: local-path-provisioner-bind
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: local-path-provisioner-role
subjects:
- kind: ServiceAccount
name: local-path-provisioner-service-account
namespace: local-path-storage
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: local-path-provisioner
namespace: local-path-storage
spec:
replicas: 1
selector:
matchLabels:
app: local-path-provisioner
template:
metadata:
labels:
app: local-path-provisioner
spec:
serviceAccountName: local-path-provisioner-service-account
containers:
- name: local-path-provisioner
image: rancher/local-path-provisioner:v0.0.35
imagePullPolicy: IfNotPresent
command:
- local-path-provisioner
- --debug
- start
- --config
- /etc/config/config.json
volumeMounts:
- name: config-volume
mountPath: /etc/config/
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CONFIG_MOUNT_PATH
value: /etc/config/
volumes:
- name: config-volume
configMap:
name: local-path-config
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: local-path
provisioner: rancher.io/local-path
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete
---
kind: ConfigMap
apiVersion: v1
metadata:
name: local-path-config
namespace: local-path-storage
data:
config.json: |-
{
"nodePathMap":[
{
"node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
"paths":["/data/kubernates"]
}
]
}
setup: |-
#!/bin/sh
set -eu
mkdir -m 0777 -p "$VOL_DIR"
teardown: |-
#!/bin/sh
set -eu
rm -rf "$VOL_DIR"
helperPod.yaml: |-
apiVersion: v1
kind: Pod
metadata:
name: helper-pod
spec:
priorityClassName: system-node-critical
tolerations:
- key: node.kubernetes.io/disk-pressure
operator: Exists
effect: NoSchedule
containers:
- name: helper-pod
image: busybox
imagePullPolicy: IfNotPresent
再创建configmap
yml
apiVersion: v1
kind: ConfigMap
metadata:
name: nacos-cm
namespace: nacos
data:
# MySQL配置
mysql.host: "10.16.10.102"
mysql.db.name: "nacos3"
mysql.port: "3306"
mysql.user: "nacos3"
mysql.password: "xxx"
# Nacos核心配置
NACOS_SERVERS: "nacos-0.nacos-headless.nacos.svc.cluster.local:8848 nacos-1.nacos-headless.nacos.svc.cluster.local:8848" #有几个节点就配置几个,多于3个就3个。
NACOS_SERVER_PORT: "8848"
PREFER_HOST_MODE: "hostname" # 使用hostname模式,配合固定域名
# JVM配置
JVM_XMS: "2g"
JVM_XMX: "2g"
JVM_XMN: "1g"
# 开启认证(生产环境建议)
NACOS_AUTH_ENABLE: "true"
NACOS_AUTH_TOKEN: "SecretKey012345678901234567890123456789012345678901234567890123456789"
NACOS_AUTH_IDENTITY_KEY: "serverIdentity"
NACOS_AUTH_IDENTITY_VALUE: "security"
创建服务:
yml
---
apiVersion: v1
kind: Service
metadata:
name: nacos-headless
namespace: nacos
labels:
app: nacos
spec:
clusterIP: None
ports:
- name: http
port: 8848
targetPort: 8848
- name: rpc
port: 9848
targetPort: 9848
- name: raft
port: 7848
targetPort: 7848
selector:
app: nacos
---
# 对外访问Service(可选)
apiVersion: v1
kind: Service
metadata:
name: nacos-lb
namespace: nacos
labels:
app: nacos
spec:
type: LoadBalancer # 关键:使用 LoadBalancer 类型 [citation:3]
ports:
- name: console
port: 8080 # 对外暴露的控制台端口
targetPort: 8080 # 注意:这里网上很多都说是指向 8848 容器端口!!!
protocol: TCP
- name: http
port: 8848 # 对外暴露的 API 端口
targetPort: 8848 # 容器内的 8848 端口
protocol: TCP
- name: grpc
port: 9848 # 对外暴露的 gRPC 端口
targetPort: 9848 # 容器内的 gRPC 端口
protocol: TCP
selector:
app: nacos
最核心的部分来了:
yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nacos
namespace: nacos
spec:
serviceName: nacos-headless
replicas: 2
selector:
matchLabels:
app: nacos
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes: ["ReadWriteOnce"]
storageClassName: local-path
resources:
requests:
storage: 2Gi
- metadata:
name: logdir
spec:
accessModes: ["ReadWriteOnce"]
storageClassName: local-path
resources:
requests:
storage: 5Gi
template:
metadata:
labels:
app: nacos
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app: nacos
# 添加 initContainer 修复权限
initContainers:
- name: volume-permissions
image: busybox
command: ['sh', '-c', 'chown -R 1000:1000 /home/nacos/data /home/nacos/logs']
volumeMounts:
- name: datadir
mountPath: /home/nacos/data
- name: logdir
mountPath: /home/nacos/logs
containers:
- name: nacos
image: nacos/nacos-server:v3.1.1
ports:
- containerPort: 8080
name: http1
- containerPort: 8848
name: http2
- containerPort: 9848
name: grpc
- containerPort: 7848
name: raft
env:
- name: MODE
value: "cluster"
- name: NACOS_REPLICAS
value: "2"
- name: NACOS_SERVER_PORT
value: "8848"
- name: SPRING_DATASOURCE_PLATFORM
value: "mysql"
- name: MYSQL_SERVICE_DB_PARAM
value: "characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false&serverTimezone=UTC"
- name: NACOS_CONFIG_NAMESPACE_COMPATIBLE_MODE
value: "false"
- name: MYSQL_SERVICE_HOST
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.host
- name: MYSQL_SERVICE_DB_NAME
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.db.name
- name: MYSQL_SERVICE_PORT
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.port
- name: MYSQL_SERVICE_USER
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.user
- name: MYSQL_SERVICE_PASSWORD
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.password
- name: NACOS_SERVERS
valueFrom:
configMapKeyRef:
name: nacos-cm
key: NACOS_SERVERS
- name: NACOS_AUTH_ENABLE
valueFrom:
configMapKeyRef:
name: nacos-cm
key: NACOS_AUTH_ENABLE
- name: NACOS_AUTH_TOKEN
valueFrom:
configMapKeyRef:
name: nacos-cm
key: NACOS_AUTH_TOKEN
- name: NACOS_AUTH_IDENTITY_KEY
value: "nacos"
- name: NACOS_AUTH_IDENTITY_VALUE
value: "nacos"
- name: JVM_XMS
valueFrom:
configMapKeyRef:
name: nacos-cm
key: JVM_XMS
- name: JVM_XMX
valueFrom:
configMapKeyRef:
name: nacos-cm
key: JVM_XMX
# livenessProbe:
# httpGet:
# path: /nacos/actuator/health
# port: 8848
# initialDelaySeconds: 60
# periodSeconds: 30
# failureThreshold: 3
# readinessProbe:
# httpGet:
# path: /nacos/actuator/health
# port: 8848
# initialDelaySeconds: 30
# periodSeconds: 10
# failureThreshold: 3
volumeMounts:
- name: datadir
mountPath: /home/nacos/data
- name: logdir
mountPath: /home/nacos/logs
重点来了:
1、上面配置中的livenessProbe和readinessProbe两个部分先要注释掉,否则会一直报错:
java
Caused by: java.lang.RuntimeException: java.net.UnknownHostException: nacos-1.nacos-headless.nacos.svc.cluster.local
at io.grpc.internal.DnsNameResolver.resolveAddresses(DnsNameResolver.java:222)
at io.grpc.internal.DnsNameResolver.doResolve(DnsNameResolver.java:281)
at io.grpc.internal.DnsNameResolver$Resolve.run(DnsNameResolver.java:318)
... 3 common frames omitted
Caused by: java.net.UnknownHostException: nacos-1.nacos-headless.nacos.svc.cluster.local
at java.base/java.net.InetAddress$CachedAddresses.get(InetAddress.java:801)
at java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1533)
at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1385)
at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1306)
at io.grpc.internal.DnsNameResolver$JdkAddressResolver.resolveAddress(DnsNameResolver.java:642)
at io.grpc.internal.DnsNameResolver.resolveAddresses(DnsNameResolver.java:218)
... 5 common frames omitted
等它启动成功,再修放开这再段,kubectl apply -f xxx.yaml就行了。
2、NACOS_CONFIG_NAMESPACE_COMPATIBLE_MODE=false
这个参数没有的话,一直会报错:
log
Caused by: org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'configMigrateService': Invocation of init method failed
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:222)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:429)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1818)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:607)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:529)
at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:339)
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:373)
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:337)
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:315)
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
at org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:254)
at org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1752)
at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1635)
at org.springframework.beans.factory.support.ConstructorResolver.resolveAutowiredArgument(ConstructorResolver.java:913)
at org.springframework.beans.factory.support.ConstructorResolver.createArgumentArray(ConstructorResolver.java:791)
... 28 common frames omitted
Caused by: java.lang.Exception: [migrate] config_info namespace migrate pre check failed
at com.alibaba.nacos.config.server.service.ConfigMigrateService.namespaceMigratePreCheck(ConfigMigrateService.java:765)
at com.alibaba.nacos.config.server.service.ConfigMigrateService.doCheckNamespaceMigrate(ConfigMigrateService.java:519)
at com.alibaba.nacos.config.server.service.ConfigMigrateService.migrate(ConfigMigrateService.java:158)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMethod.invoke(InitDestroyAnnotationBeanPostProcessor.java:457)
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:401)
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:219)
3、SPRING_DATASOURCE_PLATFORM=mysql
这个参数原文是没有的,它会导致出错:
log
[2026-03-13 11:04:10] (nacos-0 nacos) Caused by: com.alibaba.nacos.api.exception.runtime.NacosRuntimeException: errCode: 102, errMsg: dataSource or tableName is null
[2026-03-13 11:04:10] (nacos-0 nacos) at com.alibaba.nacos.plugin.datasource.MapperManager.findMapper(MapperManager.java:106)
[2026-03-13 11:04:10] (nacos-0 nacos) at com.alibaba.nacos.config.server.service.repository.extrnal.ExternalConfigInfoPersistServiceImpl.findConfigMaxId(ExternalConfigInfoPersistServiceImpl.java:720)
[2026-03-13 11:04:10] (nacos-0 nacos) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[2026-03-13 11:04:10] (nacos-0 nacos) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
[2026-03-13 11:04:10] (nacos-0 nacos) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[2026-03-13 11:04:10] (nacos-0 nacos) at java.base/java.lang.reflect.Method.invoke(Method.java:569)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:360)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:196)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.aspectj.AspectJAfterThrowingAdvice.invoke(AspectJAfterThrowingAdvice.java:64)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
[2026-03-13 11:04:10] (nacos-0 nacos) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:728)
[2026-03-13 11:04:10] (nacos-0 nacos) at com.alibaba.nacos.config.server.service.repository.extrnal.ExternalConfigInfoPersistServiceImpl$$SpringCGLIB$$0.findConfigMaxId(<generated>)
[2026-03-13 11:04:10] (nacos-0 nacos) at com.alibaba.nacos.config.server.service.dump.processor.DumpAllProcessor.process(DumpAllProcessor.java:65)
[2026-03-13 11:04:10] (nacos-0 nacos) at com.alibaba.nacos.config.server.service.dump.DumpService.dumpAllConfigInfoOnStartup(DumpService.java:270)
[2026-03-13 11:04:10] (nacos-0 nacos) at com.alibaba.nacos.config.server.service.dump.DumpService.dumpOperate(DumpService.java:224)