Docker环境部署Apache Hadoop3.1定制版

Zhu7582026-06-10 8:08

说明

在最新版本的Docker环境部署Hadoop的定制版。容器镜像是AMD64架构CPU环境使用。ARM环境的需自己定制。参考历史文章记录。

创建资源

bash 复制代码 
mkdir -p hadoop/config
cd hadoop/config

创建文件capacity-scheduler.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>yarn.scheduler.capacity.maximum-applications</name>
    <value>10000</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
    <value>0.1</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.resource-calculator</name>
    <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.queues</name>
    <value>default</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.capacity</name>
    <value>100</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
    <value>1</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
    <value>100</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.state</name>
    <value>RUNNING</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
    <value>*</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
    <value>*</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.node-locality-delay</name>
    <value>40</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.acl_application_max_priority</name>
    <value>*</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.root.default.maximum-application-lifetime</name>
    <value>-1</value>
  </property>
  <property>
    <name>yarn.scheduler.capacity.rack-locality-additional-delay</name>
    <value>-1</value>
  </property>
</configuration>

创建文件core-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>hadoop.tmp.dir</name>
    <value>/opt/hadoop/tmpdata</value>
  </property>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://namenode:9000</value> <!-- 适配compose的hostname -->
  </property>
  <property>
    <name>hadoop.proxyuser.hadoop.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hadoop.groups</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hue.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hue.groups</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hive.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.hive.groups</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.root.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.root.groups</name>
    <value>*</value>
  </property>
</configuration>

创建文件hdfs-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>/opt/hadoop/data/nn</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>/opt/hadoop/data/dn</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address</name>
    <value>namenode:9000</value> <!-- 适配compose的hostname -->
  </property>
  <property>
    <name>dfs.namenode.rpc-bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>dfs.namenode.http-bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.client.use.datanode.hostname</name>
    <value>true</value>
  </property>
  <property>
    <name>dfs.permissions.enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
  </property>
  <!-- <property>
    <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
    <value>false</value>
    <description>当namenode反向解析datanode失败时,改为false。</description>
  </property> -->
</configuration>

创建文件mapred-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <property>
    <name>yarn.app.mapreduce.am.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
  </property>
  <property>
    <name>mapreduce.map.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
  </property>
  <property>
    <name>mapreduce.reduce.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/hadoop</value>
  </property>
</configuration>

创建文件yarn-site.xml

bash 复制代码 
<?xml version="1.0"?>
<configuration>
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>resourcemanager</value> <!-- 适配compose的hostname -->
  </property>
  <property>
    <name>yarn.resourcemanager.bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>0.0.0.0:8088</value>
  </property>
  <property>
    <name>yarn.nodemanager.bind-host</name>
    <value>0.0.0.0</value>
  </property>
  <property>
    <name>yarn.nodemanager.pmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.delete.debug-delay-sec</name>
    <value>600</value>
  </property>
  <property>
    <name>yarn.nodemanager.vmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
  <property>
    <name>yarn.acl.enable</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.env-whitelist</name>
    <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
  </property>
</configuration>

创建compose.yaml

bash 复制代码 
volumes:
  hadoop-nn-data:
    name: hadoop-nn-data
  hadoop-dn-data:
    name: hadoop-dn-data
networks:
  hadoop:
    name: hadoop-net
    driver: bridge

services:
  namenode:
    image: zhuyifeiruichuang/hadoop:3.1.1
    hostname: namenode
    container_name: hadoop-namenode
    command: ["sh", "-c", "sudo mkdir -p /opt/hadoop/data/nn && sudo chown -R hadoop:hadoop /opt/hadoop/data && if [ ! -d /opt/hadoop/data/nn/current ]; then echo 'Formatting NameNode...' && hdfs namenode -format; fi && hdfs namenode"]
    ports:
      - 9870:9870
      - 9000:9000
    environment:
      ENSURE_NAMENODE_DIR: "/opt/hadoop/data/nn"
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    volumes:
      - hadoop-nn-data:/opt/hadoop/data/nn
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9870 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 60s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

  datanode:
    image: zhuyifeiruichuang/hadoop:3.1.1
    hostname: datanode
    container_name: hadoop-datanode
    command: ["sh", "-c", "sudo mkdir -p /opt/hadoop/data/dn && sudo chown -R hadoop:hadoop /opt/hadoop/data && hdfs datanode"]
    ports:
      - 9866:9866
      - 9864:9864
    environment:
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    volumes:
      - hadoop-dn-data:/opt/hadoop/data/dn
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    depends_on:
      - namenode
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9864 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 40s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

  resourcemanager:
    image: zhuyifeiruichuang/hadoop:3.1.1
    hostname: resourcemanager
    container_name: hadoop-resourcemanager
    command: ["yarn", "resourcemanager"]
    ports:
      - 8088:8088
      - 8032:8032
    environment:
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    depends_on:
      - namenode
    restart: unless-stopped
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    healthcheck:
      # test: ["CMD-SHELL", "yarn rmadmin -checkHealth || exit 1"]
      test: ["CMD-SHELL", "curl -f http://localhost:8088 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 60s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

  nodemanager:
    hostname: nodemanager
    image: zhuyifeiruichuang/hadoop:3.1.1
    container_name: hadoop-nodemanager
    command: ["yarn", "nodemanager"]
    ports:
      - 8042:8042
    environment:
      HADOOP_CONF_DIR: "/opt/hadoop/etc/hadoop"
    depends_on:
      - resourcemanager
      - namenode
    restart: unless-stopped
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ./config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml
      - ./config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml
      - ./config/capacity-scheduler.xml:/opt/hadoop/etc/hadoop/capacity-scheduler.xml
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8042 || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 40s
    networks:
      - hadoop
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G

部署

bash 复制代码 
docker compose up -d
相关推荐
镜子AI6 小时前
镜子AI 技术复盘：三条自动化链路上的“停止条件“设计
运维·人工智能·自动化
tianyuanwo6 小时前
NetBox 数据库详解：当“关系型数据库”被装进一个网络运维的领域模型里
运维·数据库·netbox
螺蛳粉 螺蛳粉7 小时前
第 05 篇：etcd 深入：Raft 共识、watch 机制与快照备份恢复实战（底座认知 05）
docker·云原生·容器·kubernetes·etcd·分布式存储·kube-apiserver
知已难寻8 小时前
03 · OpenStack的计算管理 Nova
运维·服务器·开发语言·云原生·云计算·openstack·开源软件
huainingning9 小时前
Windows syslog日志服务器搭建与模拟器对接
运维·服务器·智能路由器
say_fall9 小时前
【Linux系统编程】Ext 文件系统：磁盘与存储、文件与目录、现代计算机究竟如何寻找文件
linux·运维·服务器·数据库·c++
wacsii10 小时前
【Linux进程（一）】Linux 进程初探：操作系统如何管理进程？fork() 为什么返回两次？
linux·运维·服务器
相思难忘成疾10 小时前
从零搭建 Kubernetes 1.30 集群：kubeadm 部署一主多从实战（RHEL 9.3 + Docker + flannel）
linux·运维·kubernetes
PellyKoo10 小时前
【docker】cifs卷改smb密码后容器起不来的解决方法
运维·docker·容器
时早玖沐10 小时前
放弃 QQ 官方机器人：用 SnowLuma + AstrBot 自建 QQ 机器人
docker·开源·agent
热门推荐
01国内可直接用、免费额度/永久免费的大模型API清单（含 SiliconFlow、火山、阿里、智谱、百度、Kimi、DeepSeek、DMXAPI 等）02历年考研数学一、数学二、数学三真题试卷及答案PDF03GitHub 镜像站点04几个好用的ip纯净度检测网站05苹果iPA游戏软件资源下载网站06蜘蛛磁力 搜索引擎大全，如何使用蜘蛛磁力查找磁力链接07如何新建文件夹？ 电脑新建文件夹的4种方法08昇腾910B系列芯片完全指南：四款型号对比、服务器选型与大模型部署实战09Dify：一站式AI应用开发平台全解析与Windows部署指南10Android 屏幕适配系列开篇：核心概念、官方机制与主流方案