在k8s集群部署ApacheHadoop单节点单数据副本

说明

在kubernetes集群部署Apache Hadoop定制版本,实现单namenode,单datanode

实现了数据持久化,数据可靠性依赖k8s的pvc和pv。组件高可用依赖k8s的statefulset。

若想增强数据存储可靠性,应当给k8s集群引入可靠的storageclass,然后修改以下yaml文件中的storageclass,使存储的数据不会因node故障不可用。

配置configmap.yaml

bash 复制代码
apiVersion: v1
kind: ConfigMap
metadata:
  name: hadoop-config
data:
  core-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/hadoop/tmpdata</value>
      </property>
      <property>
        <name>fs.defaultFS</name>
        <value>hdfs://namenode-0.namenode.hadoop:9000</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hadoop.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hadoop.groups</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hue.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hue.groups</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hive.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.hive.groups</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
      </property>
      <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
      </property>
    </configuration>

  hdfs-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>dfs.namenode.name.dir</name>
        <value>/opt/hadoop/data/nn</value>
      </property>
      <property>
        <name>dfs.datanode.data.dir</name>
        <value>/opt/hadoop/data/dn</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-address</name>
        <value>namenode-0.namenode.hadoop:9000</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>dfs.namenode.http-bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>dfs.replication</name>
        <value>1</value>
      </property>
      <property>
        <name>dfs.permissions.enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
      </property>
      <property>
        <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
        <value>false</value>
        <description>当namenode反向解析datanode失败时,改为false。</description>
      </property>
    </configuration>

  yarn-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>resourcemanager-0.resourcemanager.hadoop</value>
      </property>
      <property>
        <name>yarn.resourcemanager.bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>0.0.0.0:8088</value>
      </property>
      <property>
        <name>yarn.nodemanager.bind-host</name>
        <value>0.0.0.0</value>
      </property>
      <property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>yarn.nodemanager.delete.debug-delay-sec</name>
        <value>600</value>
      </property>
      <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
      </property>
      <property>
        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
      </property>
      <property>
        <name>yarn.acl.enable</name>
        <value>false</value>
      </property>
      <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
      </property>
    </configuration>

  mapred-site.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
      </property>
      <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
      </property>
      <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
      </property>
      <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
      </property>
    </configuration>
    
  capacity-scheduler.xml: |
    <?xml version="1.0"?>
    <configuration>
      <property>
        <name>yarn.scheduler.capacity.maximum-applications</name>
        <value>10000</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
        <value>0.1</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.resource-calculator</name>
        <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.queues</name>
        <value>default</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.capacity</name>
        <value>100</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
        <value>1</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
        <value>100</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.state</name>
        <value>RUNNING</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
        <value>*</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
        <value>*</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.node-locality-delay</name>
        <value>40</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.acl_application_max_priority</name>
        <value>*</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.root.default.maximum-application-lifetime</name>
        <value>-1</value>
      </property>
      <property>
        <name>yarn.scheduler.capacity.rack-locality-additional-delay</name>
        <value>-1</value>
      </property>
    </configuration>

配置namenode.yaml

容器镜像可以改为自己构建的。

bash 复制代码
apiVersion: v1
kind: Service
metadata:
  name: namenode
  labels:
    app: hadoop-namenode
spec:
  ports:
    - port: 9870
      name: web
    - port: 9000
      name: rpc
  clusterIP: None
  selector:
    app: hadoop-namenode
---
apiVersion: v1
kind: Service
metadata:
  name: namenode-external
  labels:
    app: hadoop-namenode
spec:
  type: NodePort
  ports:
    - port: 9870
      name: web
      targetPort: 9870
      nodePort: 30070
    - port: 9000
      name: rpc
      targetPort: 9000
      nodePort: 30090
  selector:
    app: hadoop-namenode
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: namenode
spec:
  serviceName: namenode
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-namenode
  template:
    metadata:
      labels:
        app: hadoop-namenode
    spec:
      # --- 1. 初始化容器:负责权限管理、目录准备和按需格式化 ---
      initContainers:
      - name: init-namenode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        securityContext:
          runAsUser: 0
          privileged: true
        command: ["/bin/bash", "-c"]
        args:
          - |
            set -e
            echo "Step 1: Preparing directories and permissions..."
            mkdir -p /opt/hadoop/data/nn
            # 确保 hadoop 用户拥有数据目录权限
            chown -R hadoop:hadoop /opt/hadoop/data
            
            echo "Step 2: Checking if NameNode needs formatting..."
            if [ ! -f /opt/hadoop/data/nn/current/VERSION ]; then
              echo "No VERSION file found. Formatting NameNode..."
              # 使用非交互模式强制格式化
              su hadoop -c "hdfs namenode -format -nonInteractive"
              echo "Formatting completed successfully."
            else
              echo "NameNode already formatted (VERSION file exists). Skipping."
              grep -i "clusterID" /opt/hadoop/data/nn/current/VERSION
            fi
        volumeMounts:
          - name: hadoop-nn-data
            mountPath: /opt/hadoop/data/nn
            subPath: nn
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml

      # --- 2. 主容器:仅负责运行 NameNode 服务 ---
      containers:
      - name: namenode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["/bin/bash", "-c"]
        # 使用 exec 确保 namenode 是 PID 1,能够接收系统信号实现优雅停机
        args: ["exec hdfs namenode"]
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
          - name: HADOOP_LOG_DIR
            value: "/opt/hadoop/logs"
          - name: HADOOP_HEAPSIZE
            value: "1024"
        ports:
          - containerPort: 9870
            name: web
          - containerPort: 9000
            name: rpc
        # 资源限制:保证 Pod 的 QOS 等级
        resources:
          requests:
            memory: "2Gi"
            cpu: "500m"
          limits:
            memory: "4Gi"
            cpu: "1000m"
        # 健康检查:通过 JMX 接口确保 NameNode 真正可用
        readinessProbe:
          httpGet:
            path: /jmx
            port: 9870
          initialDelaySeconds: 40
          periodSeconds: 10
        livenessProbe:
          httpGet:
            path: /jmx
            port: 9870
          initialDelaySeconds: 100
          periodSeconds: 30
        volumeMounts:
          - name: hadoop-nn-data
            mountPath: /opt/hadoop/data/nn
            subPath: nn
          - name: hadoop-logs
            mountPath: /opt/hadoop/logs
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
      
      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config
        - name: hadoop-logs
          emptyDir: {}

  # --- 3. 存储定义 ---
  volumeClaimTemplates:
  - metadata:
      name: hadoop-nn-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "local"
      resources:
        requests:
          storage: 20Gi

配置datanode.yaml

bash 复制代码
apiVersion: v1
kind: Service
metadata:
  name: datanode
  labels:
    app: hadoop-datanode
spec:
  ports:
    - port: 9864
      name: web
    - port: 9866
      name: data
  clusterIP: None
  selector:
    app: hadoop-datanode
---
apiVersion: v1
kind: Service
metadata:
  name: datanode-external
  labels:
    app: hadoop-datanode
spec:
  type: NodePort
  ports:
    - port: 9864
      name: web
      targetPort: 9864
      # nodePort 留空,系统将自动分配 30000-32767 之间的随机端口
    - port: 9866
      name: data
      targetPort: 9866
  selector:
    app: hadoop-datanode
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: datanode
spec:
  serviceName: datanode
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-datanode
  template:
    metadata:
      labels:
        app: hadoop-datanode
    spec:
      securityContext:
        fsGroup: 1000

      initContainers:
      # --- 1. 等待 NameNode RPC 端口就绪 ---
      - name: wait-for-namenode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            echo "⏳ 正在检测 NameNode RPC 端口 (9000)..."
            while ! nc -z namenode-0.namenode.hadoop 9000; do
              echo "   等待 NameNode 响应中..."
              sleep 5
            done
            echo "✅ NameNode RPC 已就绪"

      # --- 2. 动态权限校准 (自动识别 hadoop 用户 UID) ---
      - name: init-permissions
        image: zhuyifeiruichuang/hadoop:3.1.1
        securityContext:
          runAsUser: 0
          privileged: true
        command: ["/bin/sh", "-c"]
        args:
          - |
            echo "🔧 正在准备数据目录并校准权限..."
            mkdir -p /opt/hadoop/data/dn
            
            # 动态检测镜像内是否存在 hadoop 用户并授权
            if id "hadoop" >/dev/null 2>&1; then
              echo "检测到 hadoop 用户,执行 chown hadoop:hadoop"
              chown -R hadoop:hadoop /opt/hadoop/data
            else
              echo "未检测到 hadoop 用户,回退使用 UID 1000"
              chown -R 1000:1000 /opt/hadoop/data
            fi
            
            # 授予 775 权限确保 DataNode 内部检查通过
            chmod -R 775 /opt/hadoop/data
            echo "✅ 权限校准完成"

      containers:
      - name: datanode
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        # 以 root 运行进程是解决 K8s 挂载卷 chmod 权限报错 (EPERM) 的最稳妥方案
        securityContext:
          runAsUser: 0
        command: ["/bin/bash", "-c"]
        args:
          - |
            export HADOOP_CONF_DIR=/opt/hadoop/etc/hadoop
            echo "🚀 启动 DataNode..."
            exec hdfs datanode
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
        ports:
          - containerPort: 9864
            name: web
          - containerPort: 9866
            name: data
        resources:
          requests:
            memory: "2Gi"
            cpu: "500m"
          limits:
            memory: "4Gi"
            cpu: "1000m"
        livenessProbe:
          httpGet:
            path: /
            port: 9864
          initialDelaySeconds: 60
          periodSeconds: 20
        readinessProbe:
          httpGet:
            path: /
            port: 9864
          initialDelaySeconds: 20
          periodSeconds: 10
        volumeMounts:
          - name: hadoop-dn-data
            mountPath: /opt/hadoop/data/dn
            subPath: dn
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
            subPath: yarn-site.xml

      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config

  # --- 存储卷模板 ---
  volumeClaimTemplates:
  - metadata:
      name: hadoop-dn-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "local"
      resources:
        requests:
          storage: 20Gi

配置nodemanager.yaml

bash 复制代码
apiVersion: v1
kind: Service
metadata:
  name: nodemanager
  labels:
    app: hadoop-nodemanager
spec:
  ports:
    - port: 8042
      name: web
  clusterIP: None
  selector:
    app: hadoop-nodemanager
---
apiVersion: v1
kind: Service
metadata:
  name: nodemanager-external
  labels:
    app: hadoop-nodemanager
spec:
  type: NodePort
  ports:
    - port: 8042
      name: web
      targetPort: 8042
  selector:
    app: hadoop-nodemanager
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nodemanager
spec:
  serviceName: nodemanager
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-nodemanager
  template:
    metadata:
      labels:
        app: hadoop-nodemanager
    spec:
      containers:
      - name: nodemanager
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["yarn", "nodemanager"]
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
        ports:
          - containerPort: 8042
        volumeMounts:
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
            subPath: yarn-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
            subPath: mapred-site.xml
      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config

配置resourcemanager.yaml

bash 复制代码
apiVersion: v1
kind: Service
metadata:
  name: resourcemanager
  labels:
    app: hadoop-resourcemanager
spec:
  ports:
    - port: 8088
      name: web
    - port: 8032
      name: rpc
  clusterIP: None
  selector:
    app: hadoop-resourcemanager
---
apiVersion: v1
kind: Service
metadata:
  name: resourcemanager-external
  labels:
    app: hadoop-resourcemanager
spec:
  type: NodePort
  ports:
    - port: 8088
      name: web
      targetPort: 8088
    - port: 8032
      name: rpc
      targetPort: 8032
  selector:
    app: hadoop-resourcemanager
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: resourcemanager
spec:
  serviceName: resourcemanager
  replicas: 1
  selector:
    matchLabels:
      app: hadoop-resourcemanager
  template:
    metadata:
      labels:
        app: hadoop-resourcemanager
    spec:
      containers:
      - name: resourcemanager
        image: zhuyifeiruichuang/hadoop:3.1.1
        imagePullPolicy: IfNotPresent
        command: ["yarn", "resourcemanager"]
        env:
          - name: HADOOP_CONF_DIR
            value: "/opt/hadoop/etc/hadoop"
        ports:
          - containerPort: 8088
          - containerPort: 8032
        volumeMounts:
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/core-site.xml
            subPath: core-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
            subPath: hdfs-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
            subPath: yarn-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
            subPath: mapred-site.xml
          - name: hadoop-config-volume
            mountPath: /opt/hadoop/etc/hadoop/capacity-scheduler.xml
            subPath: capacity-scheduler.xml
      volumes:
        - name: hadoop-config-volume
          configMap:
            name: hadoop-config

部署

bash 复制代码
kubectl create namespace hadoop
kubectl apply -f configmap.yaml -n hadoop
kubectl apply -f namenode.yaml -n hadoop
kubectl apply -f datanode.yaml -n hadoop
kubectl apply -f resourcemanager.yaml -n hadoop
kubectl apply -f nodemanager.yaml -n hadoop

部署后查询

bash 复制代码
root@master Mon Nov 24 [10:05:40] : /opt/bigdata2/hadoop/v3.1.1/v4
# kubectl get all -n hadoop 

NAME                    READY   STATUS    RESTARTS      AGE
pod/datanode-0          1/1     Running   0             9m20s
pod/namenode-0          1/1     Running   3 (14m ago)   15m
pod/nodemanager-0       1/1     Running   0             8s
pod/resourcemanager-0   1/1     Running   0             76s

NAME                      TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                      AGE
service/datanode          ClusterIP   None         <none>        9864/TCP,9866/TCP            20m
service/namenode          ClusterIP   None         <none>        9870/TCP,9000/TCP,8020/TCP   27m
service/nodemanager       ClusterIP   None         <none>        8042/TCP                     8s
service/resourcemanager   ClusterIP   None         <none>        8088/TCP,8032/TCP            7m9s

NAME                               READY   AGE
statefulset.apps/datanode          1/1     20m
statefulset.apps/namenode          1/1     27m
statefulset.apps/nodemanager       1/1     8s
statefulset.apps/resourcemanager   1/1     76s
root@master Mon Nov 24 [10:05:42] : /opt/bigdata2/hadoop/v3.1.1/v4
# kubectl get pvc -n hadoop
NAME                        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
hadoop-dn-data-datanode-0   Bound    pvc-1ce19f52-729d-4d2e-8b4a-6c064b5901fe   20Gi       RWO            local          26m
hadoop-nn-data-namenode-0   Bound    pvc-8d162dd9-c732-4337-acd2-01945a9ff929   10Gi       RWO            local          33m
相关推荐
极客先躯3 小时前
高级java每日一道面试题-2026年05月03日-实战篇[Docker]-如何实现容器化环境的数据加密?
java·运维·docker·容器·金融·加解密·高级面试
阿里云云原生7 小时前
将内核专家经验封装进 Agent:SysOM 巡检 Skill 开源,赋能你的运维自动化闭环
云原生
自律懒人8 小时前
AI应用从原型到上线的最后一公里——灵光闪应用一键部署深度实测,30+免费API + Serverless零配置发布
人工智能·云原生·开源·serverless
程序喵大人8 小时前
【C++进阶】STL容器与迭代器 - 05 map 和 set 为什么按键保持有序
开发语言·c++·容器·迭代器·stl
风曦Kisaki9 小时前
Kubernetes(K8s)笔记Day03: Pod命名空间,标签,Pod 的调度,污点与容忍度,Pod 常见状态和重启策略,Pod 生命周期
linux·运维·笔记·docker·云原生·容器·kubernetes
张忠琳9 小时前
【NVIDIA】NVIDIA k8s-device-plugin v0.19.3 资源管理器模块深度分析之三
云原生·容器·架构·kubernetes·nvidia
Chengbei1111 小时前
云安全漏洞挖掘SKILL、一站式云漏洞挖掘工具,支持S3爆破、IMDS探测、K8s检测与AK/SK权限利用
前端·人工智能·网络安全·云原生·容器·kubernetes·系统安全
Ai拆代码的曹操12 小时前
K8s 调度器 predicate 阶段揭秘:为什么你的 Pod 总是堆在同一台机器
后端·容器
Ai拆代码的曹操12 小时前
手摸手排查:Pod CrashLoopBackOff 日志丢失问题,3 层兜底方案
后端·容器
瞬间&永恒~13 小时前
【MySQL】练习5-1:配置慢查询日志
linux·运维·云原生