在k8s集群部署ApacheHadoop单节点单数据副本

Zhu7582026-07-09 8:47

说明

在kubernetes集群部署Apache Hadoop定制版本，实现单namenode，单datanode

实现了数据持久化，数据可靠性依赖k8s的pvc和pv。组件高可用依赖k8s的statefulset。

若想增强数据存储可靠性，应当给k8s集群引入可靠的storageclass，然后修改以下yaml文件中的storageclass，使存储的数据不会因node故障不可用。

配置configmap.yaml

bash 复制代码 
apiVersion: v1
kind: ConfigMap
metadata:
  name: hadoop-config
data:
  core-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/hadoop/tmpdata</value>
      </property>
      <property>
        <name>fs.defaultFS</name>
        <value>hdfs://namenode-0.namenode.hadoop:9000</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hadoop.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hadoop.groups</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hue.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hue.groups</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hive.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hive.groups</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
      </property>
    </configuration>

  hdfs-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>dfs.namenode.name.dir</name>
        <value>/opt/hadoop/data/nn</value>
      </property>
      <property>
        <name>dfs.datanode.data.dir</name>
        <value>/opt/hadoop/data/dn</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-address</name>
        <value>namenode-0.namenode.hadoop:9000</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>dfs.namenode.http-bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>dfs.replication</name>
        <value>1</value>
      </property>
      <property>
        <name>dfs.permissions.enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
      </property>
      <property>
        <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
        <value>false</value>
        <description>当namenode反向解析datanode失败时,改为false。</description>
      </property>
    </configuration>

  yarn-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>resourcemanager-0.resourcemanager.hadoop</value>
      </property>
      <property>
        <name>yarn.resourcemanager.bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>0.0.0.0:8088</value>
      </property>
      <property>
        <name>yarn.nodemanager.bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>yarn.nodemanager.delete.debug-delay-sec</name>
        <value>600</value>
      </property>
      <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
      </property>
      <property>
        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
      </property>
      <property>
        <name>yarn.acl.enable</name>
        <value>false</value>
      </property>
      <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
      </property>
    </configuration>

  mapred-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
      </property>
      <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
      </property>
      <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
      </property>
      <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
      </property>
    </configuration>
    
  capacity-scheduler.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>yarn.scheduler.capacity.maximum-applications</name>
        <value>10000</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
        <value>0.1</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.resource-calculator</name>
        <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.queues</name>
        <value>default</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.capacity</name>
        <value>100</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
        <value>1</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
        <value>100</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.state</name>
        <value>RUNNING</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
        <value>*</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
        <value>*</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.node-locality-delay</name>
        <value>40</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.acl_application_max_priority</name>
        <value>*</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.maximum-application-lifetime</name>
        <value>-1</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.rack-locality-additional-delay</name>
        <value>-1</value>
      </property>
    </configuration>

配置namenode.yaml

容器镜像可以改为自己构建的。

bash 复制代码 
apiVersion: v1
kind: Service
metadata:
  name: namenode
  labels:
    app: hadoop-namenode
spec:
  ports:
    - port: 9870
      name: web
    - port: 9000
      name: rpc
  clusterIP: None
  selector:
    app: hadoop-namenode
---
apiVersion: v1
kind: Service
metadata:
  name: namenode-external
  labels:
    app: hadoop-namenode
spec:
  type: NodePort
  ports:
    - port: 9870
      name: web
      targetPort: 9870
      nodePort: 30070
    - port: 9000
      name: rpc
      targetPort: 9000
      nodePort: 30090
  selector:
    app: hadoop-namenode
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: namenode
spec:
  serviceName: namenode
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-namenode
  template:
    metadata:
      labels:
        app: hadoop-namenode
    spec:
      # --- 1. 初始化容器：负责权限管理、目录准备和按需格式化 ---
      initContainers:
      - name: init-namenode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        securityContext:
          runAsUser: 0
          privileged: true
        command: ["/bin/bash", "-c"]
        args:
          - |
            set -e
            echo "Step 1: Preparing directories and permissions..."
            mkdir -p /opt/hadoop/data/nn
            # 确保 hadoop 用户拥有数据目录权限
            chown -R hadoop:hadoop /opt/hadoop/data
            
            echo "Step 2: Checking if NameNode needs formatting..."
            if [ ! -f /opt/hadoop/data/nn/current/VERSION ]; then
              echo "No VERSION file found. Formatting NameNode..."
              # 使用非交互模式强制格式化
              su hadoop -c "hdfs namenode -format -nonInteractive"
              echo "Formatting completed successfully."
            else
              echo "NameNode already formatted (VERSION file exists). Skipping."
              grep -i "clusterID" /opt/hadoop/data/nn/current/VERSION
            fi
        volumeMounts:
          - name: hadoop-nn-data
            mountPath: /opt/hadoop/data/nn
            subPath: nn
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml

      # --- 2. 主容器：仅负责运行 NameNode 服务 ---
      containers:
      - name: namenode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["/bin/bash", "-c"]
        # 使用 exec 确保 namenode 是 PID 1，能够接收系统信号实现优雅停机
        args: ["exec hdfs namenode"]
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
          - name: HADOOP_LOG_DIR
            value: "/opt/hadoop/logs"
          - name: HADOOP_HEAPSIZE
            value: "1024"
        ports:
          - containerPort: 9870
            name: web
          - containerPort: 9000
            name: rpc
        # 资源限制：保证 Pod 的 QOS 等级
        resources:
          requests:
            memory: "2Gi"
            cpu: "500m"
          limits:
            memory: "4Gi"
            cpu: "1000m"
        # 健康检查：通过 JMX 接口确保 NameNode 真正可用
        readinessProbe:
          httpGet:
            path: /jmx
            port: 9870
          initialDelaySeconds: 40
          periodSeconds: 10
        livenessProbe:
          httpGet:
            path: /jmx
            port: 9870
          initialDelaySeconds: 100
          periodSeconds: 30
        volumeMounts:
          - name: hadoop-nn-data
            mountPath: /opt/hadoop/data/nn
            subPath: nn
          - name: hadoop-logs
            mountPath: /opt/hadoop/logs
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
      
      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config
        - name: hadoop-logs
          emptyDir: {}

  # --- 3. 存储定义 ---
  volumeClaimTemplates:
  - metadata:
      name: hadoop-nn-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "local"
      resources:
        requests:
          storage: 20Gi

配置datanode.yaml

bash 复制代码 
apiVersion: v1
kind: Service
metadata:
  name: datanode
  labels:
    app: hadoop-datanode
spec:
  ports:
    - port: 9864
      name: web
    - port: 9866
      name: data
  clusterIP: None
  selector:
    app: hadoop-datanode
---
apiVersion: v1
kind: Service
metadata:
  name: datanode-external
  labels:
    app: hadoop-datanode
spec:
  type: NodePort
  ports:
    - port: 9864
      name: web
      targetPort: 9864
      # nodePort 留空，系统将自动分配 30000-32767 之间的随机端口
    - port: 9866
      name: data
      targetPort: 9866
  selector:
    app: hadoop-datanode
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: datanode
spec:
  serviceName: datanode
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-datanode
  template:
    metadata:
      labels:
        app: hadoop-datanode
    spec:
      securityContext:
        fsGroup: 1000

      initContainers:
      # --- 1. 等待 NameNode RPC 端口就绪 ---
      - name: wait-for-namenode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            echo "⏳ 正在检测 NameNode RPC 端口 (9000)..."
            while ! nc -z namenode-0.namenode.hadoop 9000; do
              echo "   等待 NameNode 响应中..."
              sleep 5
            done
            echo "✅ NameNode RPC 已就绪"

      # --- 2. 动态权限校准 (自动识别 hadoop 用户 UID) ---
      - name: init-permissions
        image: zhuyifeiruichuang/hadoop:3.1.1
        securityContext:
          runAsUser: 0
          privileged: true
        command: ["/bin/sh", "-c"]
        args:
          - |
            echo "🔧 正在准备数据目录并校准权限..."
            mkdir -p /opt/hadoop/data/dn
            
            # 动态检测镜像内是否存在 hadoop 用户并授权
            if id "hadoop" >/dev/null 2>&1; then
              echo "检测到 hadoop 用户，执行 chown hadoop:hadoop"
              chown -R hadoop:hadoop /opt/hadoop/data
            else
              echo "未检测到 hadoop 用户，回退使用 UID 1000"
              chown -R 1000:1000 /opt/hadoop/data
            fi
            
            # 授予 775 权限确保 DataNode 内部检查通过
            chmod -R 775 /opt/hadoop/data
            echo "✅ 权限校准完成"

      containers:
      - name: datanode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        # 以 root 运行进程是解决 K8s 挂载卷 chmod 权限报错 (EPERM) 的最稳妥方案
        securityContext:
          runAsUser: 0
        command: ["/bin/bash", "-c"]
        args:
          - |
            export HADOOP_CONF_DIR=/opt/hadoop/etc/hadoop
            echo "🚀 启动 DataNode..."
            exec hdfs datanode
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
        ports:
          - containerPort: 9864
            name: web
          - containerPort: 9866
            name: data
        resources:
          requests:
            memory: "2Gi"
            cpu: "500m"
          limits:
            memory: "4Gi"
            cpu: "1000m"
        livenessProbe:
          httpGet:
            path: /
            port: 9864
          initialDelaySeconds: 60
          periodSeconds: 20
        readinessProbe:
          httpGet:
            path: /
            port: 9864
          initialDelaySeconds: 20
          periodSeconds: 10
        volumeMounts:
          - name: hadoop-dn-data
            mountPath: /opt/hadoop/data/dn
            subPath: dn
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
            subPath: yarn-site.xml

      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config

  # --- 存储卷模板 ---
  volumeClaimTemplates:
  - metadata:
      name: hadoop-dn-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "local"
      resources:
        requests:
          storage: 20Gi

配置nodemanager.yaml

bash 复制代码 
apiVersion: v1
kind: Service
metadata:
  name: nodemanager
  labels:
    app: hadoop-nodemanager
spec:
  ports:
    - port: 8042
      name: web
  clusterIP: None
  selector:
    app: hadoop-nodemanager
---
apiVersion: v1
kind: Service
metadata:
  name: nodemanager-external
  labels:
    app: hadoop-nodemanager
spec:
  type: NodePort
  ports:
    - port: 8042
      name: web
      targetPort: 8042
  selector:
    app: hadoop-nodemanager
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nodemanager
spec:
  serviceName: nodemanager
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-nodemanager
  template:
    metadata:
      labels:
        app: hadoop-nodemanager
    spec:
      containers:
      - name: nodemanager
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["yarn", "nodemanager"]
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
        ports:
          - containerPort: 8042
        volumeMounts:
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
            subPath: yarn-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
            subPath: mapred-site.xml
      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config

配置resourcemanager.yaml

bash 复制代码 
apiVersion: v1
kind: Service
metadata:
  name: resourcemanager
  labels:
    app: hadoop-resourcemanager
spec:
  ports:
    - port: 8088
      name: web
    - port: 8032
      name: rpc
  clusterIP: None
  selector:
    app: hadoop-resourcemanager
---
apiVersion: v1
kind: Service
metadata:
  name: resourcemanager-external
  labels:
    app: hadoop-resourcemanager
spec:
  type: NodePort
  ports:
    - port: 8088
      name: web
      targetPort: 8088
    - port: 8032
      name: rpc
      targetPort: 8032
  selector:
    app: hadoop-resourcemanager
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: resourcemanager
spec:
  serviceName: resourcemanager
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-resourcemanager
  template:
    metadata:
      labels:
        app: hadoop-resourcemanager
    spec:
      containers:
      - name: resourcemanager
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["yarn", "resourcemanager"]
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
        ports:
          - containerPort: 8088
          - containerPort: 8032
        volumeMounts:
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
            subPath: yarn-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
            subPath: mapred-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/capacity-scheduler.xml
            subPath: capacity-scheduler.xml
      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config

部署

bash 复制代码 
kubectl create namespace hadoop
kubectl apply -f configmap.yaml -n hadoop
kubectl apply -f namenode.yaml -n hadoop
kubectl apply -f datanode.yaml -n hadoop
kubectl apply -f resourcemanager.yaml -n hadoop
kubectl apply -f nodemanager.yaml -n hadoop

部署后查询

bash 复制代码 
root@master Mon Nov 24 [10:05:40] : /opt/bigdata2/hadoop/v3.1.1/v4
# kubectl get all -n hadoop 

NAME                    READY   STATUS    RESTARTS      AGE
pod/datanode-0          1/1     Running   0             9m20s
pod/namenode-0          1/1     Running   3 (14m ago)   15m
pod/nodemanager-0       1/1     Running   0             8s
pod/resourcemanager-0   1/1     Running   0             76s

NAME                      TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                      AGE
service/datanode          ClusterIP   None         <none>        9864/TCP,9866/TCP            20m
service/namenode          ClusterIP   None         <none>        9870/TCP,9000/TCP,8020/TCP   27m
service/nodemanager       ClusterIP   None         <none>        8042/TCP                     8s
service/resourcemanager   ClusterIP   None         <none>        8088/TCP,8032/TCP            7m9s

NAME                               READY   AGE
statefulset.apps/datanode          1/1     20m
statefulset.apps/namenode          1/1     27m
statefulset.apps/nodemanager       1/1     8s
statefulset.apps/resourcemanager   1/1     76s
root@master Mon Nov 24 [10:05:42] : /opt/bigdata2/hadoop/v3.1.1/v4
# kubectl get pvc -n hadoop
NAME                        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
hadoop-dn-data-datanode-0   Bound    pvc-1ce19f52-729d-4d2e-8b4a-6c064b5901fe   20Gi       RWO            local          26m
hadoop-nn-data-namenode-0   Bound    pvc-8d162dd9-c732-4337-acd2-01945a9ff929   10Gi       RWO            local          33m
相关推荐
Akamai中国1 小时前
Akamai收购安全企业浏览器提供商LayerX
人工智能·云原生·云计算·云服务
摇滚侠2 小时前
云原生 Java 架构师的第一课 K8s+Docker+KubeSphere+DevOps 51-60
java·云原生·kubernetes
bukeyiwanshui2 小时前
20260703 K8s ECShop商城项目
云原生·容器·kubernetes
糖果店的幽灵2 小时前
混沌测试从入门到精通（容器与K8s）
云原生·容器·kubernetes
步步为营DotNet2 小时前
运用.NET Aspire 和 Native AOT 打造极致性能的云原生微服务
微服务·云原生·.net
蜀道山老天师14 小时前
K8s Secret 与 ConfigMap 配置管理及动态更新实战详解
云原生·容器·kubernetes
阿里云云原生16 小时前
Node.js 监控的终极补完：一次接入，全链路打通 APM、AI 观测与运行时健康
云原生·node.js
bukeyiwanshui18 小时前
20260624 k8s pod命令
云原生·容器·kubernetes
网络研究院19 小时前
Brave浏览器放大招引入容器功能，实现多账户安全隔离
网络·安全·容器·浏览器·数据·隐私
热门推荐
012026 AI 编程工具终极实战指南：Cursor vs Claude Code vs Copilot，开发者该怎么选？022026 年 AI 编程工具终极横评：Cursor vs Claude Code vs Copilot vs Windsurf03AI 编程 IDE 全景解析 2026：Agent 全面接管开发链路042026 年 AI 大模型 & AI 编程工具实战全总结05AI科技热点日报 | 2026年07月01日06GitHub 镜像站点07【AI】2026 年具身智能模型和世界模型总结082026 AI 编程工具选型横评：Cursor / Claude Code / Trae / Copilot 到底选谁（建议收藏·避坑版）09Claude Code、Codex、Cursor三分天下：2026年AI编程Agent生态全景剖析102026年7月AI圈大地震：GPT-5.6被政府限制、Claude入驻Slack、Anthropic自研芯片