Docker环境部署Apache Hadoop3.1定制版

Zhu7582026-06-10 8:08

说明

在最新版本的Docker环境部署Hadoop的定制版。容器镜像是AMD64架构CPU环境使用。ARM环境的需自己定制。参考历史文章记录。

创建资源

bash 复制代码 
mkdir -p hadoop/config
cd hadoop/config

创建文件capacity-scheduler.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>yarn.scheduler.capacity.maximum-applications</name>
    <value>10000</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
    <value>0.1</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.resource-calculator</name>
    <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.queues</name>
    <value>default</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.capacity</name>
    <value>100</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
    <value>1</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
    <value>100</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.state</name>
    <value>RUNNING</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
    <value>*</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
    <value>*</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.node-locality-delay</name>
    <value>40</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.acl_application_max_priority</name>
    <value>*</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.maximum-application-lifetime</name>
    <value>-1</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.rack-locality-additional-delay</name>
    <value>-1</value>
  </property>
</configuration>

创建文件core-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>hadoop.tmp.dir</name>
    <value>/opt/hadoop/tmpdata</value>
  </property>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://namenode:9000</value> <!-- 适配compose的hostname -->
  </property>
  <property>
    <name>hadoop.proxyuser.hadoop.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hadoop.groups</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hue.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hue.groups</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hive.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hive.groups</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.root.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.root.groups</name>
    <value>*</value>
  </property>
</configuration>

创建文件hdfs-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>/opt/hadoop/data/nn</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>/opt/hadoop/data/dn</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address</name>
    <value>namenode:9000</value> <!-- 适配compose的hostname -->
  </property>
  <property>
    <name>dfs.namenode.rpc-bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>dfs.namenode.http-bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.client.use.datanode.hostname</name>
    <value>true</value>
  </property>
  <property>
    <name>dfs.permissions.enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
  </property>
  <!-- <property>
    <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
    <value>false</value>
    <description>当namenode反向解析datanode失败时,改为false。</description>
  </property> -->
</configuration>

创建文件mapred-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <property>
    <name>yarn.app.mapreduce.am.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
  </property>
  <property>
    <name>mapreduce.map.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
  </property>
  <property>
    <name>mapreduce.reduce.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
  </property>
</configuration>

创建文件yarn-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>resourcemanager</value> <!-- 适配compose的hostname -->
  </property>
  <property>
    <name>yarn.resourcemanager.bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>0.0.0.0:8088</value>
  </property>
  <property>
    <name>yarn.nodemanager.bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>yarn.nodemanager.pmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.delete.debug-delay-sec</name>
    <value>600</value>
  </property>
  <property>
    <name>yarn.nodemanager.vmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
  <property>
    <name>yarn.acl.enable</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.env-whitelist</name>
    <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
  </property>
</configuration>

创建compose.yaml

bash 复制代码 
volumes:
  hadoop-nn-data:
    name: hadoop-nn-data
  hadoop-dn-data:
    name: hadoop-dn-data
networks:
  hadoop:
    name: hadoop-net
    driver: bridge

services:
  namenode:
    image: zhuyifeiruichuang/hadoop:3.1.1
    hostname: namenode
    container_name: hadoop-namenode
    command: ["sh", "-c", "sudo mkdir -p /opt/hadoop/data/nn && sudo chown -R hadoop:hadoop /opt/hadoop/data && if [ ! -d /opt/hadoop/data/nn/current ]; then echo 'Formatting NameNode...' && hdfs namenode -format; fi && hdfs namenode"]
    ports:
      - 9870:9870
      - 9000:9000
    environment:
      ENSURE_NAMENODE_DIR: "/opt/hadoop/data/nn"
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    volumes:
      - hadoop-nn-data:/opt/hadoop/data/nn
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9870 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 60s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

  datanode:
    image: zhuyifeiruichuang/hadoop:3.1.1
    hostname: datanode
    container_name: hadoop-datanode
    command: ["sh", "-c", "sudo mkdir -p /opt/hadoop/data/dn && sudo chown -R hadoop:hadoop /opt/hadoop/data && hdfs datanode"]
    ports:
      - 9866:9866
      - 9864:9864
    environment:
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    volumes:
      - hadoop-dn-data:/opt/hadoop/data/dn
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    depends_on:
      - namenode
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9864 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 40s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

  resourcemanager:
    image: zhuyifeiruichuang/hadoop:3.1.1
    hostname: resourcemanager
    container_name: hadoop-resourcemanager
    command: ["yarn", "resourcemanager"]
    ports:
      - 8088:8088
      - 8032:8032
    environment:
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    depends_on:
      - namenode
    restart: unless-stopped
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    healthcheck:
      # test: ["CMD-SHELL", "yarn rmadmin -checkHealth || exit 1"]
      test: ["CMD-SHELL", "curl -f http://localhost:8088 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 60s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

  nodemanager:
    hostname: nodemanager
    image: zhuyifeiruichuang/hadoop:3.1.1
    container_name: hadoop-nodemanager
    command: ["yarn", "nodemanager"]
    ports:
      - 8042:8042
    environment:
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    depends_on:
      - resourcemanager
      - namenode
    restart: unless-stopped
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8042 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 40s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

部署

bash 复制代码 
docker compose up -d
相关推荐
爱喝水的鱼丶37 分钟前
SAP-ABAP：接口与报表场景专项优化：RFC/ODATA 接口、ALV 报表的性能提升方案
运维·性能优化·接口·sap·abap·rfc·经验交流
画中有画1 小时前
自动化脚本开发最佳实践
运维·自动化
InfinitePlus3 小时前
Docker MySQL搭建一主一从
mysql·docker
江南风月4 小时前
如何使用WGCLOUD实现智能运维
运维·zabbix·运维开发·prometheus
xiaoxiangsiyan4 小时前
HCIA 网络技术基础核心模块详解
运维·网络·网络协议·tcp/ip·运维开发·网络基础
坐吃山猪6 小时前
Docker07-MySQL
mysql·docker·容器
MuMuMu12236 小时前
工业园区 VOCs 绿岛集中治理：越华环保集团项目技术方案与工程实践
大数据·运维
Wzx1980126 小时前
python沙箱和docker沙箱你选对了吗？
开发语言·python·docker
CodeDevMaster6 小时前
关于PaddleOCR-VL部署与使用说明
docker·llm
Kismet_nvi6 小时前
《Kubernetes Service 进阶、kube-proxy 与 Ingress 实战精要》
云原生·容器·kubernetes
热门推荐
01VSCode+CodeX扩展，无法启动02GitHub 镜像站点03国内可直接用、免费额度/永久免费的大模型API清单（含 SiliconFlow、火山、阿里、智谱、百度、Kimi、DeepSeek、DMXAPI 等）042026 国产 AI 大模型横评：DeepSeek、通义千问、Kimi、文心一言、星火、豆包谁更能打？05微信历史版本含下载地址（ Windows PC | 安卓 | MAC ）及设置微信不更新06历年考研数学一、数学二、数学三真题试卷及答案PDF07AI 编程 IDE 全景解析 2026：Agent 全面接管开发链路08Agnes AI 免费 API 接入指南：文本、生图、生视频，一套接口全免费09如何新建文件夹？ 电脑新建文件夹的4种方法102026年8月4日科技热点深度解析：AI大模型群雄逐鹿、卫星互联网组网提速、半导体封装材料革命